初始化
This commit is contained in:
1 parent
34474bd5fc
commit
b638b541d0
3008 files changed
+837699
No files matched your search
+70
@@ -0,0 +1,70 @@
|
||||
from .base import BaseSearchProvider, SearchResult
|
||||
from .demo_search import DemoSearchProvider
|
||||
from .duckduckgo_search import DuckDuckGoSearchProvider
|
||||
from .internet_search import InternetSearchProvider
|
||||
from .intranet_stub import IntranetSearchProvider, normalize_intranet_timeout, normalize_intranet_verify_ssl
|
||||
from .skill_search import SkillSearchProvider
|
||||
|
||||
|
||||
def get_search_provider(ai_writing_config: dict) -> BaseSearchProvider:
|
||||
"""
|
||||
config.yaml 中配置:
|
||||
ai_writing:
|
||||
search_provider: internet # internet(默认,DDG) / intranet / tavily / demo
|
||||
intranet_search_url: http://10.106.57.180:7863/knowledge_base/search_docs_es
|
||||
intranet_knowledge_base: samples
|
||||
intranet_search_timeout: 300 # 内网检索超时秒数,最少 300 秒
|
||||
intranet_verify_ssl: false # 内网 HTTPS 自签名证书场景默认关闭证书校验
|
||||
ddg_region: "cn-zh" # DuckDuckGo 区域,默认 cn-zh
|
||||
|
||||
internet 模式:DuckDuckGo 为主搜索(免 API Key),失败时 fallback 到 demo 演示素材。
|
||||
"""
|
||||
cfg = ai_writing_config or {}
|
||||
provider = cfg.get("search_provider", "internet")
|
||||
|
||||
if provider == "intranet":
|
||||
return IntranetSearchProvider(
|
||||
base_url=cfg.get("intranet_search_url", ""),
|
||||
knowledge_base_name=cfg.get("intranet_knowledge_base", "samples"),
|
||||
timeout=normalize_intranet_timeout(cfg.get("intranet_search_timeout")),
|
||||
verify_ssl=normalize_intranet_verify_ssl(cfg.get("intranet_verify_ssl")),
|
||||
)
|
||||
if provider == "tavily":
|
||||
return _FallbackSearchProvider(InternetSearchProvider(), DemoSearchProvider())
|
||||
if provider == "demo":
|
||||
return DemoSearchProvider()
|
||||
|
||||
# 默认 internet 模式:DuckDuckGo + demo fallback
|
||||
region = cfg.get("ddg_region", "cn-zh")
|
||||
return _FallbackSearchProvider(DuckDuckGoSearchProvider(region=region), DemoSearchProvider())
|
||||
|
||||
|
||||
class _FallbackSearchProvider(BaseSearchProvider):
|
||||
"""当主 provider 返回空结果时自动切换到 fallback provider。"""
|
||||
|
||||
def __init__(self, primary: BaseSearchProvider, fallback: BaseSearchProvider):
|
||||
self._primary = primary
|
||||
self._fallback = fallback
|
||||
|
||||
@property
|
||||
def errors(self) -> list[str]:
|
||||
"""聚合主/兜底 provider 的检索异常,供 researcher 节点取用。"""
|
||||
return list(getattr(self._primary, "errors", [])) + list(getattr(self._fallback, "errors", []))
|
||||
|
||||
async def search(self, query: str, max_results: int = 5, **kwargs):
|
||||
results = await self._primary.search(query, max_results=max_results, **kwargs)
|
||||
if not results:
|
||||
results = await self._fallback.search(query, max_results=max_results, **kwargs)
|
||||
return results
|
||||
|
||||
async def health_check(self) -> bool:
|
||||
return await self._primary.health_check() or await self._fallback.health_check()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"BaseSearchProvider", "SearchResult",
|
||||
"InternetSearchProvider", "IntranetSearchProvider",
|
||||
"DemoSearchProvider", "DuckDuckGoSearchProvider",
|
||||
"SkillSearchProvider",
|
||||
"get_search_provider",
|
||||
]
|
||||
+84
@@ -0,0 +1,84 @@
|
||||
import re
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass, field
|
||||
from typing import List
|
||||
|
||||
# 复合检索词里常见的「话题/类别后缀」——核心实体后面跟的修饰词。内网 ES 对
|
||||
# 「实体+话题」这类长词常检不出(如「特朗普竞选」0 条),但「实体」本身能检出
|
||||
# (「特朗普」30 条)。剥掉这些后缀回退到核心实体重试,能救回大量检索词。
|
||||
_TOPIC_SUFFIXES = (
|
||||
"竞选", "诉讼", "法律", "政策", "主张", "纲领", "动向", "动态", "影响", "分析",
|
||||
"进展", "现状", "情况", "问题", "事件", "活动", "关系", "布局", "态势", "格局",
|
||||
"趋势", "背景", "历史", "观点", "评价", "报道", "新闻", "研究", "报告", "概况",
|
||||
"走势", "前景", "措施", "方案", "计划", "战略", "改革", "发展",
|
||||
)
|
||||
|
||||
|
||||
def simplify_query(query: str) -> str:
|
||||
"""把一个「检不出」的复合检索词简化成更核心、更短的检索词,供 0 结果时重试。
|
||||
|
||||
规则(无需大模型,纯启发式):
|
||||
1. 多词(含空格)→ 取第一个词(通常是核心实体),如「美国政局 特朗普」→「美国政局」;
|
||||
2. 单词复合 → 反复剥掉结尾的话题后缀,保留核心实体,如「特朗普法律诉讼」→「特朗普」、
|
||||
「特朗普竞选」→「特朗普」。
|
||||
返回空串 = 无法进一步简化(调用方据此不再重试,避免空转)。
|
||||
"""
|
||||
s = (query or "").strip()
|
||||
if not s:
|
||||
return ""
|
||||
# 1) 多词:取第一个词
|
||||
if re.search(r"\s", s):
|
||||
first = re.split(r"\s+", s)[0].strip()
|
||||
if first and first != s and len(first) >= 2:
|
||||
return first
|
||||
# 2) 单词复合:循环剥离结尾话题后缀
|
||||
changed = s
|
||||
while True:
|
||||
stripped = changed
|
||||
for suf in sorted(_TOPIC_SUFFIXES, key=len, reverse=True):
|
||||
if stripped.endswith(suf) and len(stripped) - len(suf) >= 2:
|
||||
stripped = stripped[: -len(suf)]
|
||||
break
|
||||
if stripped == changed:
|
||||
break
|
||||
changed = stripped
|
||||
if changed != s and len(changed) >= 2:
|
||||
return changed
|
||||
return ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchResult:
|
||||
"""统一搜索结果格式 —— 内外网共用"""
|
||||
title: str
|
||||
content: str
|
||||
source: str
|
||||
published_at: str
|
||||
url: str = ""
|
||||
relevance_score: float = 0.0
|
||||
source_type: str = "internet"
|
||||
recUuid: str = ""
|
||||
extra_metadata: dict = field(default_factory=dict)
|
||||
|
||||
|
||||
class BaseSearchProvider(ABC):
|
||||
"""搜索提供者抽象基类。
|
||||
|
||||
对接内网步骤:
|
||||
1. 继承此类,创建 IntranetSearchProvider
|
||||
2. 实现 search() —— 调用内网 API,将响应映射到 SearchResult
|
||||
3. 在 config.yaml 中设置 ai_writing.search_provider: intranet
|
||||
4. 无需修改任何 Agent 代码
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
async def search(
|
||||
self,
|
||||
query: str,
|
||||
max_results: int = 5,
|
||||
date_from: str = "",
|
||||
date_to: str = "",
|
||||
) -> List[SearchResult]: ...
|
||||
|
||||
@abstractmethod
|
||||
async def health_check(self) -> bool: ...
|
||||
+183
@@ -0,0 +1,183 @@
|
||||
"""离线演示搜索提供者 —— 当无法访问互联网/内网时返回模拟素材,供测试完整写作流程。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from typing import List
|
||||
|
||||
from .base import BaseSearchProvider, SearchResult
|
||||
|
||||
# 演示素材库:按话题分类,用于关键词模糊匹配
|
||||
_DEMO_CORPUS = [
|
||||
{
|
||||
"keywords": ["新能源", "电动车", "汽车", "ev", "electric"],
|
||||
"items": [
|
||||
{
|
||||
"title": "2025年新能源汽车市场规模突破万亿,渗透率达45%",
|
||||
"content": "据中国汽车工业协会数据,2025年前三季度新能源汽车累计销量超过850万辆,同比增长32%,市场渗透率达到45.3%。比亚迪、吉利、长城等头部企业份额持续集中,纯电与插混比例趋近5:5。",
|
||||
"source": "中国汽车工业协会",
|
||||
"published_at": "2025-10-15",
|
||||
"relevance_score": 0.95,
|
||||
},
|
||||
{
|
||||
"title": "固态电池量产进入倒计时:多家车企宣布2026年装车计划",
|
||||
"content": "宁德时代、比亚迪、丰田同步推进固态电池量产。固态电池能量密度可达400Wh/kg,较现有液态锂电池提升60%,续航里程有望突破1000公里。成本仍是主要瓶颈,预计量产初期每度电成本高出30%。",
|
||||
"source": "电池中国网",
|
||||
"published_at": "2025-09-20",
|
||||
"relevance_score": 0.88,
|
||||
},
|
||||
{
|
||||
"title": "充电基础设施缺口分析:农村地区桩车比仅1:12",
|
||||
"content": "截至2025年8月,全国充电桩保有量达1120万根,城市地区桩车比约1:2.8,但农村及县域地区桩车比仅1:12,制约新能源汽车下沉市场。国家电网计划2026年前新增农村快充站3万座。",
|
||||
"source": "中国充电联盟",
|
||||
"published_at": "2025-08-30",
|
||||
"relevance_score": 0.82,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"keywords": ["人工智能", "ai", "大模型", "llm", "机器学习", "深度学习"],
|
||||
"items": [
|
||||
{
|
||||
"title": "国内大模型进入「百亿参数普惠化」阶段,推理成本下降90%",
|
||||
"content": "2025年第三季度,国内主要大模型服务商将百亿参数模型推理价格降至每百万token不足1元,较2024年同期下降90%。开源生态加速,Qwen3、DeepSeek-V3等模型在多项基准测试中超越GPT-4o。",
|
||||
"source": "中国信通院",
|
||||
"published_at": "2025-10-08",
|
||||
"relevance_score": 0.93,
|
||||
},
|
||||
{
|
||||
"title": "企业AI应用落地白皮书:制造业ROI最高达300%",
|
||||
"content": "调研覆盖500家企业的白皮书显示,AI在质检、排产、预测性维护三个场景ROI最高,部分制造企业ROI超300%。阻碍因素:数据孤岛(67%)、人才不足(54%)、安全合规(48%)。",
|
||||
"source": "麦肯锡中国",
|
||||
"published_at": "2025-09-12",
|
||||
"relevance_score": 0.87,
|
||||
},
|
||||
{
|
||||
"title": "AI写作助手用户规模达2.3亿,内容质量争议持续",
|
||||
"content": "国内AI写作工具月活用户突破2.3亿,覆盖学术论文、营销文案、新闻报道等场景。争议焦点:AI生成内容的版权归属、事实准确性(幻觉率约15%)、对写作从业者就业影响。",
|
||||
"source": "艾瑞咨询",
|
||||
"published_at": "2025-10-01",
|
||||
"relevance_score": 0.79,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"keywords": ["碳中和", "双碳", "绿色", "低碳", "碳排放", "可再生能源", "光伏", "风电"],
|
||||
"items": [
|
||||
{
|
||||
"title": "光伏装机量突破900GW,度电成本降至0.18元",
|
||||
"content": "国家能源局数据显示,截至2025年9月,全国光伏累计装机量达912GW,占总装机的28%。光伏度电成本已降至0.18元,首次低于煤电基准价,推动更多省份出现「光伏平价」区域。",
|
||||
"source": "国家能源局",
|
||||
"published_at": "2025-10-12",
|
||||
"relevance_score": 0.92,
|
||||
},
|
||||
{
|
||||
"title": "碳市场价格突破200元/吨,倒逼高耗能行业转型",
|
||||
"content": "全国碳排放权交易市场碳价已突破200元/吨,较2024年底上涨65%。水泥、电解铝、造纸等高耗能行业面临显著成本压力,碳减排投入年均增速达40%。",
|
||||
"source": "上海环境能源交易所",
|
||||
"published_at": "2025-09-28",
|
||||
"relevance_score": 0.85,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"keywords": ["医疗", "健康", "药物", "生物", "医药", "病"],
|
||||
"items": [
|
||||
{
|
||||
"title": "AI辅助诊断落地三甲医院,影像科漏诊率降低23%",
|
||||
"content": "北京协和、上海瑞金等20家三甲医院数据显示,引入AI辅助诊断系统后,肺结节、眼底病变早期漏诊率降低23%,放射科医生平均阅片时间减少40%。监管层面,NMPA已批准38款AI医疗器械。",
|
||||
"source": "医学界",
|
||||
"published_at": "2025-09-18",
|
||||
"relevance_score": 0.91,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"keywords": ["经济", "GDP", "贸易", "出口", "产业", "制造"],
|
||||
"items": [
|
||||
{
|
||||
"title": "2025年前三季度GDP同比增长5.1%,消费贡献率提升至58%",
|
||||
"content": "国家统计局公布,2025年前三季度GDP同比增长5.1%,最终消费对经济增长贡献率达58%,较去年同期提升8个百分点。工业增加值增长5.8%,其中装备制造业增长9.2%,成重要支撑。",
|
||||
"source": "国家统计局",
|
||||
"published_at": "2025-10-18",
|
||||
"relevance_score": 0.90,
|
||||
},
|
||||
{
|
||||
"title": "跨境电商出口额突破2.4万亿元,东南亚成最大增量市场",
|
||||
"content": "商务部数据显示,2025年前三季度跨境电商出口额达2.4万亿元,同比增长28%。东南亚市场增速最快(+45%),中东、南美紧随其后。Temu、速卖通、Shopee三平台合计占出口份额约62%。",
|
||||
"source": "商务部",
|
||||
"published_at": "2025-10-05",
|
||||
"relevance_score": 0.84,
|
||||
},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
# 通用兜底素材(与任意查询都能凑合搭配)
|
||||
_FALLBACK_ITEMS = [
|
||||
{
|
||||
"title": "行业研究方法论:如何构建有效的一手与二手数据体系",
|
||||
"content": "高质量行业报告需整合政府统计、上市公司财报、行业协会数据与专家访谈。一手调研样本量建议≥200份,二手数据优先选用权威机构发布的年度报告与季度白皮书,并注明数据时效性与口径差异。",
|
||||
"source": "产业研究院",
|
||||
"published_at": "2025-08-15",
|
||||
"relevance_score": 0.55,
|
||||
},
|
||||
{
|
||||
"title": "政策解读:2025年产业发展专项支持资金管理办法",
|
||||
"content": "工信部发布2025年产业发展专项资金管理办法,重点支持「专精特新」企业数字化改造、绿色制造和关键技术攻关。单项目最高补贴1000万元,要求申报企业研发投入不低于营收3%。",
|
||||
"source": "工信部",
|
||||
"published_at": "2025-07-20",
|
||||
"relevance_score": 0.50,
|
||||
},
|
||||
{
|
||||
"title": "市场竞争格局分析:头部集中与长尾分散并存",
|
||||
"content": "多个新兴行业呈现「哑铃型」竞争格局:头部3-5家企业合计市占率超60%,长尾中小企业依靠细分场景差异化生存,中间层企业面临最大竞争压力。并购整合预期升温,2025年战略性并购交易同比增长35%。",
|
||||
"source": "普华永道",
|
||||
"published_at": "2025-09-01",
|
||||
"relevance_score": 0.48,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def _score_topic(query: str, keywords: list[str]) -> int:
|
||||
"""简单关键词匹配打分"""
|
||||
q = query.lower()
|
||||
return sum(1 for kw in keywords if kw in q)
|
||||
|
||||
|
||||
class DemoSearchProvider(BaseSearchProvider):
|
||||
"""离线演示搜索提供者:根据查询关键词从内置语料库返回模拟素材。
|
||||
|
||||
仅用于开发测试;生产环境应配置 intranet 提供者。
|
||||
"""
|
||||
|
||||
async def search(self, query: str, max_results: int = 5, **kwargs) -> List[SearchResult]:
|
||||
best_topic = max(_DEMO_CORPUS, key=lambda t: _score_topic(query, t["keywords"]), default=None)
|
||||
best_score = _score_topic(query, best_topic["keywords"]) if best_topic else 0
|
||||
|
||||
items = []
|
||||
if best_score > 0 and best_topic:
|
||||
items.extend(best_topic["items"])
|
||||
|
||||
# 用通用素材补齐
|
||||
for fb in _FALLBACK_ITEMS:
|
||||
if len(items) >= max_results:
|
||||
break
|
||||
items.append(fb)
|
||||
|
||||
results = []
|
||||
for i, it in enumerate(items[:max_results]):
|
||||
url_hash = hashlib.md5(it["title"].encode()).hexdigest()[:8]
|
||||
results.append(SearchResult(
|
||||
title=it["title"],
|
||||
content=it["content"],
|
||||
source=it["source"],
|
||||
published_at=it["published_at"],
|
||||
url=f"https://demo.internal/{url_hash}",
|
||||
relevance_score=it["relevance_score"],
|
||||
source_type="demo",
|
||||
recUuid=f"demo-{i:03d}",
|
||||
))
|
||||
return results
|
||||
|
||||
async def health_check(self) -> bool:
|
||||
return True
|
||||
+81
@@ -0,0 +1,81 @@
|
||||
"""DuckDuckGo 搜索提供者 —— 免 API Key 互联网搜索。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import List
|
||||
|
||||
from .base import BaseSearchProvider, SearchResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DuckDuckGoSearchProvider(BaseSearchProvider):
|
||||
"""使用 DuckDuckGo 进行互联网搜索,无需 API Key。"""
|
||||
|
||||
def __init__(self, region: str = "cn-zh", max_timeout: int = 120):
|
||||
self.region = region
|
||||
self.max_timeout = max_timeout
|
||||
# 收集本次运行的检索异常,供 researcher 节点取出并提示用户
|
||||
self.errors: list[str] = []
|
||||
|
||||
async def search(
|
||||
self,
|
||||
query: str,
|
||||
max_results: int = 5,
|
||||
date_from: str = "",
|
||||
date_to: str = "",
|
||||
) -> List[SearchResult]:
|
||||
try:
|
||||
results = await asyncio.to_thread(
|
||||
self._sync_search, query, max_results
|
||||
)
|
||||
return results
|
||||
except Exception as e:
|
||||
logger.warning("DuckDuckGo search failed for query=%r: %s", query, e)
|
||||
self.errors.append(f"{type(e).__name__}: {e}")
|
||||
return []
|
||||
|
||||
def _sync_search(self, query: str, max_results: int) -> List[SearchResult]:
|
||||
try:
|
||||
from ddgs import DDGS
|
||||
except ImportError:
|
||||
logger.error("ddgs not installed. Run: uv add ddgs")
|
||||
self.errors.append("ImportError: ddgs 未安装")
|
||||
return []
|
||||
|
||||
ddgs = DDGS(timeout=self.max_timeout)
|
||||
try:
|
||||
raw = ddgs.text(
|
||||
query,
|
||||
region=self.region,
|
||||
safesearch="moderate",
|
||||
max_results=max_results,
|
||||
)
|
||||
items = list(raw) if raw else []
|
||||
except Exception as e:
|
||||
logger.warning("DDGS.text failed: %s", e)
|
||||
self.errors.append(f"{type(e).__name__}: {e}")
|
||||
return []
|
||||
|
||||
results = []
|
||||
for item in items:
|
||||
url = item.get("href", item.get("link", ""))
|
||||
source = url.split("/")[2] if url and "/" in url else "web"
|
||||
results.append(SearchResult(
|
||||
title=item.get("title", ""),
|
||||
content=item.get("body", item.get("snippet", ""))[:500],
|
||||
source=source,
|
||||
published_at=item.get("published", ""),
|
||||
url=url,
|
||||
relevance_score=0.75,
|
||||
source_type="internet",
|
||||
))
|
||||
return results
|
||||
|
||||
async def health_check(self) -> bool:
|
||||
try:
|
||||
results = await self.search("test", max_results=1)
|
||||
return len(results) > 0
|
||||
except Exception:
|
||||
return False
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import List
|
||||
|
||||
from .base import BaseSearchProvider, SearchResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class InternetSearchProvider(BaseSearchProvider):
|
||||
"""互联网搜索,使用项目已有的 Tavily 搜索工具。"""
|
||||
|
||||
async def search(
|
||||
self,
|
||||
query: str,
|
||||
max_results: int = 5,
|
||||
date_from: str = "",
|
||||
date_to: str = "",
|
||||
) -> List[SearchResult]:
|
||||
try:
|
||||
results = await asyncio.to_thread(self._sync_search, query, max_results)
|
||||
return results
|
||||
except Exception as e:
|
||||
logger.warning("internet search failed for query=%r: %s", query, e)
|
||||
return []
|
||||
|
||||
def _sync_search(self, query: str, max_results: int) -> List[SearchResult]:
|
||||
from deerflow.config import get_app_config
|
||||
try:
|
||||
from tavily import TavilyClient
|
||||
except ImportError:
|
||||
logger.warning("tavily not installed, falling back to demo search")
|
||||
return []
|
||||
|
||||
config = get_app_config().get_tool_config("web_search")
|
||||
api_key = None
|
||||
if config is not None and "api_key" in config.model_extra:
|
||||
api_key = config.model_extra.get("api_key")
|
||||
|
||||
if not api_key:
|
||||
logger.warning("Tavily API key not configured, falling back to demo search")
|
||||
return []
|
||||
|
||||
client = TavilyClient(api_key=api_key)
|
||||
res = client.search(query, max_results=max_results)
|
||||
|
||||
results = []
|
||||
for item in res.get("results", []):
|
||||
content = item.get("content", "")
|
||||
results.append(SearchResult(
|
||||
title=item.get("title", ""),
|
||||
content=content[:500],
|
||||
source=item.get("url", "").split("/")[2] if "/" in item.get("url", "") else "web",
|
||||
published_at=item.get("published_date", ""),
|
||||
url=item.get("url", ""),
|
||||
relevance_score=item.get("score", 0.0),
|
||||
source_type="internet",
|
||||
))
|
||||
return results
|
||||
|
||||
async def health_check(self) -> bool:
|
||||
try:
|
||||
results = await self.search("test", max_results=1)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
+282
@@ -0,0 +1,282 @@
|
||||
"""内网知识库搜索适配器 —— 对接 ES 检索接口 search_docs_es。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
|
||||
from .base import BaseSearchProvider, SearchResult, simplify_query
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 内网知识库默认检索接口与知识库名(可经 config.yaml 的 ai_writing 段覆盖)
|
||||
_DEFAULT_ENDPOINT = "http://10.106.57.180:7863/knowledge_base/search_docs_es"
|
||||
_DEFAULT_KB = "samples"
|
||||
# 内网检索最少等待 5 分钟;该接口经常慢返回,低于这个值会造成素材收集误判为空。
|
||||
_MIN_TIMEOUT_SECONDS = 300
|
||||
_DEFAULT_TIMEOUT_SECONDS = 600
|
||||
# 内网 HTTPS 检索服务常使用自签名/内部 CA 证书,默认跳过证书校验避免 SSL 证书链错误。
|
||||
_DEFAULT_VERIFY_SSL = False
|
||||
# 单条原始响应体最大保留长度,避免把超大响应整段灌进前端时间线
|
||||
_RAW_SNIPPET_LIMIT = 2000
|
||||
|
||||
|
||||
def normalize_intranet_timeout(timeout: object, default: int = _DEFAULT_TIMEOUT_SECONDS) -> int:
|
||||
"""返回内网检索超时秒数,并强制不低于 5 分钟。"""
|
||||
try:
|
||||
value = int(timeout)
|
||||
except (TypeError, ValueError):
|
||||
value = default
|
||||
return max(value, _MIN_TIMEOUT_SECONDS)
|
||||
|
||||
|
||||
def normalize_intranet_verify_ssl(value: object, default: bool = _DEFAULT_VERIFY_SSL) -> bool:
|
||||
"""返回内网检索是否校验 SSL 证书,兼容 config.yaml 里的 bool / 字符串写法。"""
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if value is None:
|
||||
return default
|
||||
if isinstance(value, (int, float)):
|
||||
return bool(value)
|
||||
if isinstance(value, str):
|
||||
normalized = value.strip().lower()
|
||||
if normalized in {"1", "true", "yes", "on"}:
|
||||
return True
|
||||
if normalized in {"0", "false", "no", "off"}:
|
||||
return False
|
||||
return default
|
||||
|
||||
|
||||
def _pick(d: dict, keys: tuple) -> str:
|
||||
"""从结果项里按候选字段名取第一个非空值(两套接口字段名不同,做并集兼容)。"""
|
||||
for k in keys:
|
||||
v = d.get(k)
|
||||
if isinstance(v, str) and v.strip():
|
||||
return v.strip()
|
||||
if v not in (None, "") and not isinstance(v, (dict, list)):
|
||||
return str(v).strip()
|
||||
return ""
|
||||
|
||||
|
||||
def _raw_snippet(body: str) -> str:
|
||||
"""Return a bounded raw response snippet for diagnostics."""
|
||||
body = (body or "").strip()
|
||||
if not body:
|
||||
return ""
|
||||
if len(body) > _RAW_SNIPPET_LIMIT:
|
||||
return body[:_RAW_SNIPPET_LIMIT] + "…(响应体过长,已截断)"
|
||||
return body
|
||||
|
||||
|
||||
class IntranetSearchProvider(BaseSearchProvider):
|
||||
"""内网知识库搜索接口适配器。
|
||||
|
||||
接口:POST {endpoint}
|
||||
请求体:{"query": "...", "knowledge_base_name": "samples", "ts": "yes"}
|
||||
响应体:[{"page_content": "...", "metadata": {m_title, content, recUuid, m_publish, source1}}]
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
base_url: str = "",
|
||||
knowledge_base_name: str = _DEFAULT_KB,
|
||||
timeout: int = _DEFAULT_TIMEOUT_SECONDS,
|
||||
verify_ssl: bool = _DEFAULT_VERIFY_SSL,
|
||||
**_ignored,
|
||||
):
|
||||
self.endpoint = base_url or _DEFAULT_ENDPOINT
|
||||
self.knowledge_base_name = knowledge_base_name or _DEFAULT_KB
|
||||
# 内网 ES 接口很慢,兜底检索超时默认 10 分钟,且无论配置如何至少 5 分钟。
|
||||
self.timeout = normalize_intranet_timeout(timeout)
|
||||
self.verify_ssl = normalize_intranet_verify_ssl(verify_ssl)
|
||||
# 收集本次运行的检索异常,供 researcher 节点取出并提示用户(ES 波动等)
|
||||
self.errors: list[str] = []
|
||||
# 解析失败时收集的原始响应体,供前端「错误返回结果」折叠展示
|
||||
self.raw_responses: list[str] = []
|
||||
# 每一次真实 HTTP 调用的诊断明细。search() 可能会因 0 结果触发简化词重试,
|
||||
# 因此一个业务关键词可能对应多条 calls。
|
||||
self.calls: list[dict] = []
|
||||
|
||||
def _record_raw(self, body: str) -> None:
|
||||
"""收集解析失败 / 结构异常时的原始响应体,供前端折叠查看。"""
|
||||
snippet = _raw_snippet(body)
|
||||
if snippet:
|
||||
self.raw_responses.append(snippet)
|
||||
|
||||
async def search(self, query: str, max_results: int = 5, **kwargs) -> list[SearchResult]:
|
||||
results = await self._search_once(query, max_results)
|
||||
# 0 结果兜底:复合检索词常检不出(如「特朗普竞选」),换成更核心的检索词重试一次
|
||||
# (「特朗普竞选」→「特朗普」)。仅在首检为空、且能简化出不同的词时才重试。
|
||||
if not results:
|
||||
simpler = simplify_query(query)
|
||||
if simpler and simpler != query:
|
||||
logger.info("intranet search empty for %r, retry with simplified %r", query, simpler)
|
||||
results = await self._search_once(simpler, max_results)
|
||||
return results
|
||||
|
||||
async def _search_once(self, query: str, max_results: int = 5) -> list[SearchResult]:
|
||||
import httpx
|
||||
|
||||
payload = {
|
||||
"query": query,
|
||||
"knowledge_base_name": self.knowledge_base_name,
|
||||
"ts": "yes",
|
||||
}
|
||||
# 先拿到原始响应文本,JSON 解析失败时把它原样回传前端便于定位接口返回了什么
|
||||
raw_text = ""
|
||||
started = time.monotonic()
|
||||
call: dict = {
|
||||
"query": query,
|
||||
"endpoint": self.endpoint,
|
||||
"knowledge_base_name": self.knowledge_base_name,
|
||||
"max_results": max_results,
|
||||
"status": "pending",
|
||||
"ok": False,
|
||||
"count": 0,
|
||||
"verify_ssl": self.verify_ssl,
|
||||
}
|
||||
# 排查用:每次真实调用都打一行请求明细(端点 / 知识库 / 检索词 / 超时),
|
||||
# 内网复现时直接看后端日志即可知道"到底请求了哪个地址、用什么参数"。
|
||||
logger.info(
|
||||
"[intranet-search] POST endpoint=%s kb=%s timeout=%ss verify_ssl=%s query=%r",
|
||||
self.endpoint, self.knowledge_base_name, self.timeout, self.verify_ssl, query,
|
||||
)
|
||||
try:
|
||||
# trust_env=False:**关键**。内网 ES 是内网地址,但本部署的模型走公网,
|
||||
# 服务器通常设了 HTTP_PROXY/HTTPS_PROXY 出公网;httpx 默认 trust_env=True
|
||||
# 会把这个内网请求也套用公网代理 → 代理到不了内网 IP → 连接被重置/空回复,
|
||||
# 表现为「接口异常」。强制忽略代理 env、直连内网(与 multi_agent.py 的回环
|
||||
# 调用同款修法)。
|
||||
# verify=False:内网 HTTPS 常见自签名/内部 CA 证书;这里仅跳过本内网检索端点的证书校验。
|
||||
async with httpx.AsyncClient(timeout=self.timeout, trust_env=False, verify=self.verify_ssl) as client:
|
||||
resp = await client.post(self.endpoint, json=payload)
|
||||
call["http_status"] = getattr(resp, "status_code", None)
|
||||
resp.raise_for_status()
|
||||
raw_text = resp.text
|
||||
data = resp.json()
|
||||
except httpx.HTTPStatusError as e:
|
||||
# 非 2xx:接口有响应但状态码异常,把响应体一并暴露给前端
|
||||
body_for_log = e.response.text if e.response is not None else ""
|
||||
logger.warning(
|
||||
"[intranet-search] HTTP error endpoint=%s query=%r status=%s body=%s",
|
||||
self.endpoint, query, getattr(e.response, "status_code", None), _raw_snippet(body_for_log),
|
||||
)
|
||||
message = f"{type(e).__name__}: {e}"
|
||||
self.errors.append(message)
|
||||
call.update({
|
||||
"status": "http_error",
|
||||
"error": message,
|
||||
"http_status": getattr(e.response, "status_code", None),
|
||||
})
|
||||
snippet = _raw_snippet(body_for_log)
|
||||
if snippet:
|
||||
call["raw_response"] = snippet
|
||||
self._record_raw(body_for_log)
|
||||
return []
|
||||
except Exception as e:
|
||||
# 连接失败 / 超时 / JSON 解析失败等;JSON 解析失败时 raw_text 已是接口响应体。
|
||||
# 打出异常类型,便于区分 ConnectError / ConnectTimeout / ReadTimeout /
|
||||
# RemoteProtocolError(空回复) / JSONDecodeError(返回非 JSON) 等具体根因。
|
||||
logger.warning(
|
||||
"[intranet-search] request failed endpoint=%s query=%r error=%s: %s",
|
||||
self.endpoint, query, type(e).__name__, e,
|
||||
)
|
||||
message = f"{type(e).__name__}: {e}"
|
||||
self.errors.append(message)
|
||||
call.update({"status": "error", "error": message})
|
||||
snippet = _raw_snippet(raw_text)
|
||||
if snippet:
|
||||
call["raw_response"] = snippet
|
||||
self._record_raw(raw_text)
|
||||
return []
|
||||
finally:
|
||||
call["elapsed_ms"] = int((time.monotonic() - started) * 1000)
|
||||
# 非 pending 表示已经在异常分支写好;成功分支会在 return 前覆盖并追加。
|
||||
if call.get("status") != "pending":
|
||||
self.calls.append(call)
|
||||
|
||||
# 正常情况下接口直接返回数组;兼容包了一层 data/results 的结构(含工作正常的
|
||||
# knowledge-search-v2:{"success":true,"results":[...]})。
|
||||
if isinstance(data, list):
|
||||
items = data
|
||||
elif isinstance(data, dict):
|
||||
if "success" in data:
|
||||
call["response_success"] = bool(data.get("success"))
|
||||
if "count" in data:
|
||||
call["response_count"] = data.get("count")
|
||||
# 接口显式标失败时把整体回传前端便于定位
|
||||
if data.get("success") is False:
|
||||
message = f"接口返回 success=false:{str(data.get('error') or '')[:200]}"
|
||||
self.errors.append(message)
|
||||
call.update({"status": "success_false", "error": message})
|
||||
snippet = _raw_snippet(raw_text)
|
||||
if snippet:
|
||||
call["raw_response"] = snippet
|
||||
self.calls.append(call)
|
||||
self._record_raw(raw_text)
|
||||
return []
|
||||
items = data.get("data") or data.get("results") or data.get("documents") or []
|
||||
else:
|
||||
# 返回结构既不是数组也不是已知 dict 包装:视为异常,把原始响应回传前端
|
||||
logger.warning("intranet search unexpected response type for query=%r: %s", query, type(data).__name__)
|
||||
message = f"返回结构异常:期望数组,实际为 {type(data).__name__}"
|
||||
self.errors.append(message)
|
||||
call.update({"status": "bad_shape", "error": message})
|
||||
snippet = _raw_snippet(raw_text)
|
||||
if snippet:
|
||||
call["raw_response"] = snippet
|
||||
self.calls.append(call)
|
||||
self._record_raw(raw_text)
|
||||
return []
|
||||
|
||||
logger.info(
|
||||
"[intranet-search] OK endpoint=%s query=%r http_status=%s raw_items=%s",
|
||||
self.endpoint, query, call.get("http_status"), len(items) if isinstance(items, list) else 0,
|
||||
)
|
||||
results: list[SearchResult] = []
|
||||
for i, item in enumerate(items[:max_results]):
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
# 字段名做并集兼容两套接口:
|
||||
# 老 ES: {page_content, metadata:{m_title, content, recUuid, m_publish, source1}}
|
||||
# knowledge-search-v2:{title, content_preview, url, recUuid, source_category, publish_time, relevance_score}
|
||||
md = item.get("metadata") if isinstance(item.get("metadata"), dict) else {}
|
||||
merged = {**md, **item} if md else item
|
||||
content = _pick(merged, ("content", "content_preview", "page_content", "text", "summary"))
|
||||
title = _pick(merged, ("title", "m_title", "name", "doc_title"))
|
||||
if not content and not title:
|
||||
continue
|
||||
try:
|
||||
score = float(merged.get("relevance_score"))
|
||||
except (TypeError, ValueError):
|
||||
# 接口未返回相关度分值:按返回顺序赋递减分以保留排序
|
||||
score = max(0.5, 0.95 - i * 0.03)
|
||||
results.append(SearchResult(
|
||||
title=title or "(无标题)",
|
||||
content=str(content)[:500],
|
||||
source=_pick(merged, ("source_category", "source1", "source", "source_name", "kb_name")) or "内网知识库",
|
||||
published_at=_pick(merged, ("publish_time", "m_publish", "published_at", "date")),
|
||||
# 工作接口直接给了 url;老 ES 没有 url(由前端依据 recUuid 拼接),保持空串
|
||||
url=_pick(merged, ("url",)),
|
||||
relevance_score=score,
|
||||
source_type="intranet",
|
||||
recUuid=_pick(merged, ("recUuid", "rec_uuid", "id")),
|
||||
))
|
||||
call.update({
|
||||
"status": "ok" if results else "empty",
|
||||
"ok": True,
|
||||
"count": len(results),
|
||||
"item_count": len(items) if isinstance(items, list) else 0,
|
||||
})
|
||||
if not results:
|
||||
snippet = _raw_snippet(raw_text)
|
||||
if snippet:
|
||||
call["raw_response"] = snippet
|
||||
self.calls.append(call)
|
||||
return results
|
||||
|
||||
async def health_check(self) -> bool:
|
||||
try:
|
||||
await self.search("测试", max_results=1)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
+229
@@ -0,0 +1,229 @@
|
||||
"""技能选择器:根据用户意图 + 输入,从启用的 Skills 里挑出按相关度排序的若干个。
|
||||
|
||||
注意:这个模块**只负责选择技能,不直接产出素材**。
|
||||
真正的素材产出由 researcher 节点用每个 skill 的描述去编排真实 web/intranet 检索。
|
||||
|
||||
流程(对应素材确认卡上的"技能检索"按钮):
|
||||
1) list_skills 事件 —— 列出当前启用的全部 skill
|
||||
2) skill_select_chunk 事件 —— LLM 流式打字 + <think></think> 分离(前端打字机展示)
|
||||
3) select_skills 事件 —— LLM 给出的排序结果(结构化字段,写到 SkillStepsCard 里)
|
||||
|
||||
下游 researcher 节点拿到 selected_skills 后,**按顺序**:
|
||||
- 用 skill.description + user_query 生成针对性 query
|
||||
- 调真实 web search provider 执行
|
||||
- 累积素材达到阈值就停(推 stop_early)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from typing import Callable, List, Optional
|
||||
|
||||
from langchain_core.messages import HumanMessage, SystemMessage
|
||||
from langchain_core.runnables import RunnableConfig
|
||||
|
||||
from ..nodes._utils import (
|
||||
ThinkTagParser,
|
||||
_content_to_text,
|
||||
get_model,
|
||||
parse_json,
|
||||
reasoning_delta,
|
||||
safe_get_stream_writer,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
_SELECT_SKILLS_SYSTEM = """你是一名情报研究员,需要从可用「技能」中挑出对当前检索任务最有帮助的若干个,并按相关度从高到低排序。
|
||||
技能是一段封装好的指令 / 知识 / 检索策略,每个技能有 name 和 description。
|
||||
|
||||
【输出要求】
|
||||
- 只输出一个 JSON 对象,不要 markdown 代码块。
|
||||
- 必须使用英文双引号。
|
||||
- selected_skills 中的 name 必须严格来自下方"可用技能"列表,不要编造,按相关度从高到低排序。
|
||||
- 如果用户输入里明确点名了某个技能(按 name 完全匹配),把它排在第一位。
|
||||
- 至少选 1 个,最多选 {max_select} 个。
|
||||
|
||||
【完整示例】
|
||||
{{"selected_skills": ["platform-sales-data", "industry-report"], "reason": "用户想查各平台销量,平台销量技能最相关"}}"""
|
||||
|
||||
_SELECT_SKILLS_USER = """写作意图:{user_intent}
|
||||
用户补充输入:{user_query}
|
||||
|
||||
可用技能(共 {skill_count} 个):
|
||||
{skill_list}
|
||||
|
||||
请从中按相关度从高到低选出最多 {max_select} 个技能。
|
||||
只输出 JSON:
|
||||
{{"selected_skills": ["name1", "name2"], "reason": "为什么选这些 + 排序依据"}}"""
|
||||
|
||||
|
||||
EmitStepFn = Callable[[dict], None]
|
||||
|
||||
|
||||
class SkillPicker:
|
||||
"""从启用的 Skills 中流式选择按相关度排序的若干个供下游编排使用。
|
||||
|
||||
构造参数:
|
||||
config: LangGraph runnable config(用于创建 model)
|
||||
model_name: 指定使用的模型,None 则用 configurable.model_name
|
||||
emit_step: 推送时间线步骤事件(list_skills / select_skills)
|
||||
max_select: LLM 单次最多挑几个技能
|
||||
user_query: 用户在素材确认卡输入框写的自然语言
|
||||
candidates: 候选技能池。None = 全部启用的技能(旧行为);
|
||||
传入列表(如素材收集专家 agent 配置的技能)则只在其中匹配排序
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: RunnableConfig,
|
||||
*,
|
||||
model_name: Optional[str] = None,
|
||||
emit_step: Optional[EmitStepFn] = None,
|
||||
max_select: int = 5,
|
||||
user_query: str = "",
|
||||
candidates: Optional[list] = None,
|
||||
) -> None:
|
||||
self._config = config
|
||||
self._model_name = model_name
|
||||
self._emit_step = emit_step or (lambda _step: None)
|
||||
self._max_select = max_select
|
||||
self._user_query = (user_query or "").strip()
|
||||
self._candidates = candidates
|
||||
self._errors: list[str] = []
|
||||
|
||||
@property
|
||||
def errors(self) -> list[str]:
|
||||
return list(self._errors)
|
||||
|
||||
def _load_enabled_skills(self) -> list:
|
||||
if self._candidates is not None:
|
||||
skills = list(self._candidates)
|
||||
self._emit_step({
|
||||
"type": "list_skills",
|
||||
"message": f"素材收集专家已配置 {len(skills)} 个检索技能",
|
||||
"skill_names": [s.name for s in skills],
|
||||
})
|
||||
return skills
|
||||
try:
|
||||
from deerflow.skills import get_or_new_skill_storage
|
||||
storage = get_or_new_skill_storage()
|
||||
skills = storage.load_skills(enabled_only=True)
|
||||
except Exception as e:
|
||||
logger.exception("ai_writing.skill_search: 加载启用技能失败")
|
||||
self._errors.append(f"skill list error: {e}")
|
||||
skills = []
|
||||
self._emit_step({
|
||||
"type": "list_skills",
|
||||
"message": f"发现 {len(skills)} 个可用技能",
|
||||
"skill_names": [s.name for s in skills],
|
||||
})
|
||||
return skills
|
||||
|
||||
async def pick(self, user_intent: str) -> List[dict]:
|
||||
"""流式选择并排序候选技能,返回 [{skill, reason}, ...],按相关度降序。
|
||||
|
||||
过程中通过 custom 通道推送 ``skill_select_chunk`` 事件给前端打字机;
|
||||
结束后再推一条 ``select_skills`` 结构化事件。
|
||||
"""
|
||||
skills = self._load_enabled_skills()
|
||||
if not skills:
|
||||
self._emit_step({
|
||||
"type": "select_skills",
|
||||
"message": "当前没有启用的技能可用",
|
||||
"skill_names": [],
|
||||
})
|
||||
return []
|
||||
|
||||
skill_list_text = "\n".join(
|
||||
f"- {s.name}: {s.description or '(无描述)'}" for s in skills
|
||||
)
|
||||
|
||||
fallback = {"selected_skills": [s.name for s in skills[: self._max_select]], "reason": ""}
|
||||
|
||||
# list_skills 之后立刻发一个"分析中"提示步骤,避免 LLM 第一个 chunk 到来前
|
||||
# 用户看到空窗期(某些模型 astream 在头部 buffer 时间较长)
|
||||
self._emit_step({
|
||||
"type": "match_skills",
|
||||
"message": "正在让模型从候选技能中筛选最合适的…",
|
||||
})
|
||||
|
||||
# 流式 LLM 调用:与 intent_parser / writer_outline 同套机制
|
||||
writer = safe_get_stream_writer()
|
||||
parser = ThinkTagParser()
|
||||
full_text = ""
|
||||
|
||||
_chunk_count = {"n": 0}
|
||||
|
||||
def _emit_chunk(text: str, is_thinking: bool) -> None:
|
||||
# 只在首个 chunk 打一条 timing,避免逐 token 刷屏;用来和 skill_search_step
|
||||
# 的 [skill-timing] 对比"选择打字"与"步骤事件"谁先到。
|
||||
if _chunk_count["n"] == 0:
|
||||
logger.info("[skill-timing] t=%.3f first skill_select_chunk", time.monotonic())
|
||||
_chunk_count["n"] += 1
|
||||
writer({
|
||||
"type": "skill_select_chunk",
|
||||
"text": text,
|
||||
"is_thinking": is_thinking,
|
||||
"agent_name": "素材收集专家",
|
||||
})
|
||||
|
||||
# 立刻推一个空 chunk,让前端 streaming view 立即挂载(占位 + 闪烁光标),
|
||||
# 即使后续 LLM 首个 chunk 到达较慢,UI 也不再静默
|
||||
_emit_chunk("", False)
|
||||
|
||||
try:
|
||||
model = get_model(self._config, model_name=self._model_name)
|
||||
async for chunk in model.astream([
|
||||
SystemMessage(content=_SELECT_SKILLS_SYSTEM.format(max_select=self._max_select)),
|
||||
HumanMessage(content=_SELECT_SKILLS_USER.format(
|
||||
user_intent=user_intent or "(无明确写作意图)",
|
||||
user_query=self._user_query or "(无)",
|
||||
skill_count=len(skills),
|
||||
skill_list=skill_list_text,
|
||||
max_select=self._max_select,
|
||||
)),
|
||||
]):
|
||||
# 推理模型的思考走 reasoning_content(content 为空):作为思考实时推送
|
||||
reasoning = reasoning_delta(chunk)
|
||||
if reasoning:
|
||||
_emit_chunk(reasoning, True)
|
||||
delta = _content_to_text(chunk.content)
|
||||
if not delta:
|
||||
continue
|
||||
for is_thinking, text in parser.feed(delta):
|
||||
if not is_thinking:
|
||||
full_text += text
|
||||
_emit_chunk(text, is_thinking)
|
||||
except Exception as e:
|
||||
logger.exception("ai_writing.skill_search: 流式选择技能失败")
|
||||
self._errors.append(f"skill select error: {e}")
|
||||
|
||||
result = parse_json(full_text, fallback) if full_text else fallback
|
||||
|
||||
wanted_names = result.get("selected_skills") or []
|
||||
if not isinstance(wanted_names, list):
|
||||
wanted_names = []
|
||||
by_name = {s.name: s for s in skills}
|
||||
reason = result.get("reason", "") or ""
|
||||
chosen: list[dict] = []
|
||||
for name in wanted_names:
|
||||
if not isinstance(name, str):
|
||||
continue
|
||||
sk = by_name.get(name.strip())
|
||||
if sk and sk.name not in {c["skill"].name for c in chosen}:
|
||||
chosen.append({"skill": sk, "reason": reason})
|
||||
if len(chosen) >= self._max_select:
|
||||
break
|
||||
|
||||
self._emit_step({
|
||||
"type": "select_skills",
|
||||
"message": f"按相关度排序,候选 {len(chosen)} 个技能(将按顺序使用)",
|
||||
"skill_names": [c["skill"].name for c in chosen],
|
||||
"reason": reason,
|
||||
})
|
||||
return chosen
|
||||
|
||||
|
||||
# 保留旧名字让 search/__init__.py 的导出不报错;研究节点不再用它
|
||||
SkillSearchProvider = SkillPicker
|
||||
Reference in new issue
Block a user