Merge branch 'ji' into master

合并 ji 的功能:开放问答接口(/api/open/chat[/stream] 回传生成文件 + 临时不留存)、
技能地址管理、智能体详情「最近对话」等。
冲突 frontend StrategyRoutes.tsx 已解决:同时保留 admin/roundtable-diagnostics(master)
与 admin/skill-addresses(ji) 两个管理员路由。

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
吉鑫升andClaude Opus 4.8 committed 2026-06-22 19:39:35 +08:00
commit fbec222eb4
38 files changed
+3365 -94

No files matched your search

@@ -50,6 +50,7 @@ from app.gateway.routers import (
models,
multi_agent,
notifications,
open_chat,
parallel_agent_tasks,
parallel_agents,
positions,
@@ -69,6 +70,7 @@ from app.gateway.routers import (
runs,
scheduled_tasks,
sensitive_words,
skill_addresses,
skills,
suggestions,
system_settings,
@@ -1359,6 +1361,9 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
# Sensitive-word map (敏感词管理 / 脱敏词) at /api/sensitive-words
app.include_router(sensitive_words.router)
# Skill URL/IP batch replace (技能地址管理) at /api/skill-addresses
app.include_router(skill_addresses.router)
# 大屏绘制智能体独立页面会话(按 taskId 存取,不分权)at /api/dashboard-sessions
app.include_router(dashboard_sessions.router)
@@ -1456,6 +1461,9 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
# Stateless Runs API (stream/wait without a pre-existing thread)
app.include_router(runs.router)
# 开放问答接口 /api/open/chat —— 无需登录,指定模型 + 智能体做问答,可关闭思考模式
app.include_router(open_chat.router)
# --- ?????????? /page/workspace/roundtable/planning?---
# ??? app/gateway/routers/{multi_agent,intent,recommend}.py?
# ??? frontend-web/docs/multi-agent-backend-dev.md
@@ -35,6 +35,9 @@ _PUBLIC_PATH_PREFIXES: tuple[str, ...] = (
"/redoc",
"/openapi.json",
"/api/public/",
# 开放问答接口:无需登录即可指定模型 + 智能体做问答(见 routers/open_chat.py)。
# 以 "default" 用户桶运行,只暴露受控的问答能力。
"/api/open/",
)
# Exact auth paths that are public (login/register/status check).
@@ -52,6 +52,10 @@ def should_check_csrf(request: Request) -> bool:
# PUT/POST returns 403 even though auth was bypassed earlier in the chain.
if path.startswith("/api/public/"):
return False
# 开放问答接口同样是匿名、无会话的(见 auth_middleware._PUBLIC_PATH_PREFIXES),
# 双提交 Cookie 既无 token 可比、也无会话可被伪造,豁免之。
if path.startswith("/api/open/"):
return False
return True
@@ -0,0 +1,432 @@
"""开放问答接口(无需登录)。
给外部系统一个最简单的 HTTP 入口来「调用本项目的智能体做问答」:
- 可指定 **模型**(``model_name``,对应 ``config.yaml`` 中 ``models[]`` 的 name)。
- 可指定 **智能体**(``agent_id``,对应 ``.deer-flow/agents/<id>/``;留空 = 默认 lead agent)。
- 可关闭 **大模型思考模式**(``thinking_enabled=false``,默认即关闭)。内网 vLLM/Qwen
模型仅靠 ``thinking_enabled=False`` 往往关不掉思考(其 config 只声明
``supports_thinking: true``),本接口在关闭思考时会把
``thinking_force_disabled`` 一并下发,经
``create_chat_model(force_disable_thinking=True)`` 强制注入
``extra_body.chat_template_kwargs.enable_thinking=false``(及 GLM 的
``thinking.type=disabled``)——与「生成标题」时关闭思考的做法一致。
该路由前缀 ``/api/open/`` 已在 ``auth_middleware`` 白名单里,**绕过登录鉴权**,
以 ``"default"`` 用户桶运行,只暴露受控的问答能力(无文件/线程持久化)。
请求示例::
POST /api/open/chat
{
"message": "用一句话介绍一下你自己",
"model_name": "deepseek-ai/DeepSeek-V4-Flash",
"agent_id": null,
"thinking_enabled": false
}
返回::
{"answer": "...", "model_name": "...", "agent_id": null,
"thinking_enabled": false, "thread_id": "open-...."}
"""
from __future__ import annotations
import asyncio
import base64
import json
import logging
import re
import shutil
import uuid
from collections.abc import AsyncIterator
from pathlib import Path
from typing import Any, Literal
from fastapi import APIRouter, HTTPException
from fastapi.responses import StreamingResponse
from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, SystemMessage
from pydantic import BaseModel, Field
from deerflow.agents.lead_agent.agent import make_lead_agent
from deerflow.config.agents_config import list_custom_agents, validate_agent_id
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/open", tags=["open"])
_THINK_TAG_RE = re.compile(r"<think>[\s\S]*?</think>", re.IGNORECASE)
class OpenChatMessage(BaseModel):
"""多轮历史中的一条消息。"""
role: Literal["user", "assistant", "system"] = "user"
content: str
class OpenChatRequest(BaseModel):
"""开放问答请求体。
``message`` 与 ``messages`` 二选一:单轮直接传 ``message``;需要带上下文时传
``messages``(完整历史,本接口本身无状态、不落库)。
"""
message: str | None = Field(default=None, description="单轮问题文本;与 messages 二选一")
messages: list[OpenChatMessage] | None = Field(default=None, description="多轮历史;与 message 二选一")
model_name: str | None = Field(default=None, description="models[] 中的模型名;留空用默认模型")
agent_name: str | None = Field(default=None, description="智能体名称(中文显示名);留空用默认 lead agent")
agent_id: str | None = Field(default=None, description="智能体 id;一般不用,优先级高于 agent_name")
thinking_enabled: bool = Field(default=False, description="是否开启思考;默认 False = 关闭思考(内网模型也会被强制关闭)")
strip_think: bool = Field(default=True, description="关闭思考时,从答案里剥离残留的 <think>…</think> 块")
thread_id: str | None = Field(default=None, description="可选;仅用于文件隔离,留空自动生成")
recursion_limit: int = Field(default=100, ge=1, le=300, description="智能体单次问答的最大步数")
include_files: bool = Field(default=True, description="是否随结果回传智能体在 outputs 目录生成的文件(名称+内容)")
class OpenChatFile(BaseModel):
"""智能体本次运行在 /mnt/user-data/outputs 下生成的一个文件。"""
name: str # outputs/ 下的相对路径,如 "报告.md"
virtual_path: str # 沙箱虚拟路径,如 "/mnt/user-data/outputs/报告.md"
encoding: Literal["text", "base64"] = "text" # text=可直接用的文本;base64=二进制需解码
content: str
class OpenChatResponse(BaseModel):
answer: str
model_name: str | None
agent_name: str | None
agent_id: str | None
thinking_enabled: bool
thread_id: str
files: list[OpenChatFile] = Field(default_factory=list, description="本次生成的输出文件(含内容)")
def _resolve_agent_id(agent_id: str | None, agent_name: str | None) -> str | None:
"""把 agent_id / 中文名称 解析成运行 id。
优先级:显式 agent_id > agent_name(按 .deer-flow/agents/*/config.yaml 的 name 匹配)。
都没传 = None(默认 lead agent)。名称匹配先精确、再忽略大小写/首尾空格。
"""
if agent_id and agent_id.strip():
try:
return validate_agent_id(agent_id)
except ValueError as exc:
raise HTTPException(status_code=422, detail=f"非法的 agent_id:{exc}") from exc
if not agent_name or not agent_name.strip():
return None
target = agent_name.strip()
agents = list_custom_agents()
for a in agents:
if (a.name or "").strip() == target:
return a.id
lowered = target.lower()
for a in agents:
if (a.name or "").strip().lower() == lowered:
return a.id
raise HTTPException(status_code=404, detail=f"未找到名称为「{agent_name}」的智能体。")
def _build_lc_messages(body: OpenChatRequest) -> list[BaseMessage]:
"""把请求体转成 LangChain 消息列表。"""
if body.messages:
out: list[BaseMessage] = []
for m in body.messages:
text = (m.content or "").strip()
if not text:
continue
if m.role == "assistant":
out.append(AIMessage(content=m.content))
elif m.role == "system":
out.append(SystemMessage(content=m.content))
else:
out.append(HumanMessage(content=m.content))
if out:
return out
if body.message and body.message.strip():
return [HumanMessage(content=body.message)]
raise HTTPException(status_code=422, detail="必须提供非空的 'message' 或 'messages'。")
def _extract_text(content: Any) -> str:
"""从 AIMessage.content(str 或 block 列表)抽取纯文本。"""
if isinstance(content, str):
return content
if isinstance(content, list):
parts: list[str] = []
for block in content:
if isinstance(block, str):
parts.append(block)
elif isinstance(block, dict):
text_value = block.get("text")
if isinstance(text_value, str):
parts.append(text_value)
return "".join(parts) if parts else ""
return ""
def _extract_answer(result: Any, *, strip_think: bool) -> str:
"""从 ainvoke 的最终状态里取最后一条 AI 文本。"""
messages = (result or {}).get("messages") if isinstance(result, dict) else None
answer = ""
if isinstance(messages, list):
for msg in reversed(messages):
if isinstance(msg, AIMessage):
text = _extract_text(msg.content).strip()
if text:
answer = text
break
if strip_think and answer:
answer = _THINK_TAG_RE.sub("", answer).strip()
return answer
_OUTPUTS_VIRTUAL_PREFIX = "/mnt/user-data/outputs"
_MAX_OUTPUT_FILES = 20
_MAX_OUTPUT_FILE_BYTES = 5 * 1024 * 1024 # 单文件 5MB 上限,避免把大二进制塞进 JSON 响应
def _collect_output_files(state: Any, *, include: bool) -> list[OpenChatFile]:
"""读取本次运行在沙箱 outputs 目录生成的文件,随结果一并回传。
开放问答是无状态的(不持久化线程),客户端没法用 artifacts 路由按线程取件;但本端点
就跑在服务端进程里,沙箱把生成的文件落在 ``state['thread_data']['outputs_path']``,
所以这里直接扫描该目录,把文件内容带回(文本按 utf-8,二进制按 base64)。
受 ``_MAX_OUTPUT_FILES`` / ``_MAX_OUTPUT_FILE_BYTES`` 限制。
"""
if not include or not isinstance(state, dict):
return []
thread_data = state.get("thread_data") or {}
outputs_path = thread_data.get("outputs_path")
if not outputs_path:
return []
base = Path(outputs_path)
if not base.is_dir():
return []
try:
entries = sorted((p for p in base.rglob("*") if p.is_file()), key=lambda p: p.as_posix())
except OSError:
return []
files: list[OpenChatFile] = []
for entry in entries:
if len(files) >= _MAX_OUTPUT_FILES:
break
try:
if entry.stat().st_size > _MAX_OUTPUT_FILE_BYTES:
continue
data = entry.read_bytes()
except OSError:
continue
rel = entry.relative_to(base).as_posix()
try:
content = data.decode("utf-8")
encoding: Literal["text", "base64"] = "text"
except UnicodeDecodeError:
content = base64.b64encode(data).decode("ascii")
encoding = "base64"
files.append(OpenChatFile(name=rel, virtual_path=f"{_OUTPUTS_VIRTUAL_PREFIX}/{rel}", encoding=encoding, content=content))
return files
def _cleanup_ephemeral_thread(state: Any, thread_id: str | None) -> None:
"""删除本次开放问答的临时线程目录(open-*),落实"聊天记录不存储"。
生成文件已在 _collect_output_files 读进内存(随响应回传),这里把磁盘上整个临时线程
目录清掉,避免无状态调用在服务端堆积残留。仅对 ``open-*`` 线程生效,best-effort。
"""
if not thread_id or not str(thread_id).startswith("open-"):
return
thread_data = state.get("thread_data") if isinstance(state, dict) else None
outputs_path = (thread_data or {}).get("outputs_path")
if not outputs_path:
return
try:
p = Path(outputs_path).resolve()
# outputs_path 形如 .../threads/<thread_id>/user-data/outputs;只删到 <thread_id> 这层
thread_root = next((parent for parent in p.parents if parent.name == thread_id), None)
if thread_root is not None and thread_root.exists():
shutil.rmtree(thread_root, ignore_errors=True)
except Exception as exc: # noqa: BLE001 — 清理失败不影响返回结果
logger.warning("清理临时线程目录失败 (%s): %s", thread_id, exc)
# 流式响应每隔这么久没有真实事件就发一个 SSE 注释心跳,持续制造下行字节,
# 让前置反向代理(nginx proxy_read_timeout 默认 60s 等)不会把长任务判定成空闲超时。
_HEARTBEAT_SECONDS = 15.0
def _build_run_context(body: OpenChatRequest) -> tuple[dict[str, Any], str | None, str]:
"""把请求体翻译成 (config, agent_id, thread_id)。两个端点共用,确保关思考等行为一致。"""
agent_id = _resolve_agent_id(body.agent_id, body.agent_name)
thread_id = body.thread_id or f"open-{uuid.uuid4()}"
configurable: dict[str, Any] = {
"thread_id": thread_id,
"model_name": body.model_name,
"thinking_enabled": body.thinking_enabled,
# 关闭思考时强制下发,确保内网模型也真的关掉(见模块 docstring)。
"thinking_force_disabled": not body.thinking_enabled,
# 一次性问答:跳过自动起标题(省一次 LLM 调用)、不挂定时任务工具。
"is_scheduled_run": True,
# 临时/无状态运行:跳过记忆与知识库的读写(不召回、不留存、不自动沉淀),
# 做到"通过本接口的聊天记录不存储"(见 _build_middlewares 的 is_ephemeral)。
"ephemeral": True,
"memory_injection_enabled": False,
}
if agent_id:
configurable["agent_id"] = agent_id
config: dict[str, Any] = {"configurable": configurable, "recursion_limit": body.recursion_limit}
return config, agent_id, thread_id
def _sse(event: str | None, data: Any) -> str:
"""序列化一条 SSE 消息。event=None 时只发 data 行。"""
payload = json.dumps(data, ensure_ascii=False)
head = f"event: {event}\n" if event else ""
return f"{head}data: {payload}\n\n"
@router.post("/chat", response_model=OpenChatResponse)
async def open_chat(body: OpenChatRequest) -> OpenChatResponse:
"""指定模型 + 智能体做一次问答(无状态、可关闭思考)。
同步阻塞版:整段调研跑完才返回。**长任务(联网调研)请改用 /chat/stream**——
本端点期间不产生任何下行字节,容易被前置反代的 proxy_read_timeout 掐成 504。
"""
lc_messages = _build_lc_messages(body)
config, agent_id, thread_id = _build_run_context(body)
try:
agent = make_lead_agent(config)
result = await agent.ainvoke({"messages": lc_messages}, config=config)
except Exception as exc: # noqa: BLE001 — 开放接口需把内部错误归一成 502
logger.exception("open_chat 调用智能体失败 (model=%s, agent=%s)", body.model_name, agent_id)
raise HTTPException(status_code=502, detail=f"智能体调用失败:{exc}") from exc
answer = _extract_answer(result, strip_think=body.strip_think and not body.thinking_enabled)
files = _collect_output_files(result, include=body.include_files)
_cleanup_ephemeral_thread(result, thread_id) # 取完文件即清理临时目录,不在服务端留存
if not answer and not files:
raise HTTPException(status_code=502, detail="智能体未返回任何文本答案或文件。")
return OpenChatResponse(
answer=answer,
model_name=body.model_name,
agent_name=body.agent_name,
agent_id=agent_id,
thinking_enabled=body.thinking_enabled,
thread_id=thread_id,
files=files,
)
async def _open_chat_event_stream(body: OpenChatRequest) -> AsyncIterator[str]:
"""驱动智能体并产出 SSE 事件流。
- 边跑边转发 AI 文本增量(``event: delta``,仅供实时观看,客户端可忽略);
- 没有真实事件时每 ``_HEARTBEAT_SECONDS`` 发一个心跳注释行保活;
- 全部跑完后用 ``event: result`` 给出最终(已按需剥离 <think> 的)答案;
- 出错则 ``event: error``。
用「后台生产任务 + 队列 + 超时取队列」的方式把「智能体进度」与「定时心跳」解耦,
所以即便某次工具调用静默很久,心跳仍照常下行,连接不会被反代判空闲。
"""
lc_messages = _build_lc_messages(body)
config, agent_id, thread_id = _build_run_context(body)
strip_think = body.strip_think and not body.thinking_enabled
queue: asyncio.Queue[tuple[str, Any]] = asyncio.Queue()
final_state: dict[str, Any] = {}
async def produce() -> None:
nonlocal final_state
try:
agent = make_lead_agent(config)
async for mode, chunk in agent.astream({"messages": lc_messages}, config=config, stream_mode=["values", "messages"]):
if mode == "values" and isinstance(chunk, dict):
final_state = chunk
await queue.put(("progress", {"messages": len(chunk.get("messages") or [])}))
elif mode == "messages":
msg = chunk[0] if isinstance(chunk, tuple) and chunk else chunk
# 仅转发 AI 正文增量;DeepSeek 等把思考放在 additional_kwargs.reasoning_content,
# 不在 content 里,所以这里天然不会泄漏 <think>。
if isinstance(msg, AIMessage):
text = _extract_text(getattr(msg, "content", "")).strip()
if text:
await queue.put(("delta", {"text": text}))
except Exception as exc: # noqa: BLE001 — 开放接口需把内部错误归一成 error 事件
logger.exception("open_chat_stream 调用智能体失败 (model=%s, agent=%s)", body.model_name, agent_id)
await queue.put(("error", {"detail": f"智能体调用失败:{exc}"}))
finally:
await queue.put(("__end__", None))
task = asyncio.create_task(produce())
try:
while True:
try:
event, data = await asyncio.wait_for(queue.get(), timeout=_HEARTBEAT_SECONDS)
except asyncio.TimeoutError:
yield ": keepalive\n\n" # SSE 注释行,客户端忽略,仅用于保活
continue
if event == "__end__":
break
if event == "error":
yield _sse("error", data)
return
yield _sse(event, data)
answer = _extract_answer(final_state, strip_think=strip_think)
files = _collect_output_files(final_state, include=body.include_files)
_cleanup_ephemeral_thread(final_state, thread_id) # 取完文件即清理临时目录,不在服务端留存
if not answer and not files:
yield _sse("error", {"detail": "智能体未返回任何文本答案或文件。"})
return
yield _sse(
"result",
{
"answer": answer,
"model_name": body.model_name,
"agent_name": body.agent_name,
"agent_id": agent_id,
"thinking_enabled": body.thinking_enabled,
"thread_id": thread_id,
"files": [f.model_dump() for f in files],
},
)
finally:
if not task.done():
task.cancel()
@router.post("/chat/stream")
async def open_chat_stream(body: OpenChatRequest) -> StreamingResponse:
"""流式版 /chat:边跑边发心跳保活,跑完用 ``event: result`` 给最终答案。
解决两件事:(1) 长任务(联网调研、报告)同步阻塞会被前置反代 proxy_read_timeout
掐成 504;流式持续有字节下行,代理不再判定空闲超时。(2) 客户端只需读到
``event: result`` 解析 ``answer`` 即可,中间的 ``delta``/``progress``/心跳都可忽略。
"""
# 入参校验放在流外:非法请求直接抛 4xx,而不是开了流再报错。
_build_lc_messages(body)
return StreamingResponse(
_open_chat_event_stream(body),
media_type="text/event-stream",
headers={
"Cache-Control": "no-cache",
"Connection": "keep-alive",
# 关掉 nginx 对该响应的缓冲,确保 SSE/心跳实时下行(即便前置 nginx 默认开缓冲)。
"X-Accel-Buffering": "no",
},
)
@router.get("/agents")
async def open_list_agents() -> dict[str, Any]:
"""列出可用智能体的名称(方便知道 agent_name 该填什么)。"""
agents = list_custom_agents()
items = [{"name": a.name, "id": a.id} for a in agents if (a.name or "").strip()]
return {"agents": items, "count": len(items)}
@@ -67,24 +67,11 @@ async def list_recommended_questions(
) -> RecommendedQuestionsListResponse:
store = _get_store(request)
rows = await store.list_all()
if not all:
# Filter by the caller's 岗位 (position) visibility. ``None`` =
# unrestricted (no skill/question tag binding → show all default
# questions); a set = show only the questions carrying the bound tags.
sync = getattr(request.app.state, "position_sync", None)
if sync is not None:
try:
visible = await sync.resolve_visible(_current_user_id(request), "recommended_question")
except Exception:
logger.debug("recommended-question visibility resolve failed", exc_info=True)
visible = None
if visible is not None:
rows = [r for r in rows if r["id"] in visible]
responses = [RecommendedQuestionResponse(**r) for r in rows]
# Bulk-attach the tags assigned to each question so the management list can
# display them and filter by 标签. Best-effort: a missing/failing tag store
# never breaks the question list.
# display them and the 岗位 filter below can tell tagged from untagged.
# Best-effort: a missing/failing tag store never breaks the question list.
tag_store = getattr(request.app.state, "tag_store", None)
if tag_store is not None and responses:
try:
@@ -95,6 +82,24 @@ async def list_recommended_questions(
for resp in responses:
resp.tags = assignments.get(resp.id, [])
if not all:
# Filter by the caller's 岗位 (position) visibility:
# * ``set`` — the position binds recommended_question 标签 → show only
# the questions carrying those bound tags.
# * ``None`` — 默认岗位 / 无标签绑定 → show only **untagged** questions.
# 打标的题目只在绑定了对应标签的岗位展示,不再泄漏到默认岗位。
sync = getattr(request.app.state, "position_sync", None)
if sync is not None:
try:
visible = await sync.resolve_visible(_current_user_id(request), "recommended_question")
except Exception:
logger.debug("recommended-question visibility resolve failed", exc_info=True)
visible = None
if visible is not None:
responses = [r for r in responses if r.id in visible]
else:
responses = [r for r in responses if not r.tags]
if tag_ids:
wanted = set(tag_ids)
responses = [r for r in responses if any(t.get("id") in wanted for t in r.tags)]
@@ -0,0 +1,713 @@
"""技能地址(URL/IP)批量替换 API (技能地址管理).
Scans the **running** skills directory (``skills/{public,custom}``) — the files
the agent actually reads — for ``http(s)://`` URLs and IPv4 (``ip[:port]``)
addresses, aggregates each unique address with every place it appears (across
``.md`` and ``.py``, across files), and lets an admin rewrite them in bulk.
Why this shape: the same address routinely lives in both a skill's ``SKILL.md``
(documented endpoint) and its ``scripts/*.py`` (the actual call), so the natural
unit is *"one address → replace all of its occurrences at once"*. The pure
scan/replace logic lives in ``deerflow.skills.address_scan`` (harness layer,
substring-safe span replacement); this router only walks the tree, enforces
admin auth + path safety, backs up before writing, and refreshes the skills
system-prompt cache so edits take effect immediately.
**Scale / availability (150+ skills):** every filesystem walk / read / write is
offloaded to a worker thread via ``asyncio.to_thread`` so the event loop is
never blocked while scanning or rewriting — the Gateway stays responsive. The
streaming endpoints additionally emit NDJSON progress (``已处理 X / 共 N``) so the
UI can render a progress bar for both the scan and the apply phases.
Routes (prefix ``/api/skill-addresses``, **admin only**):
GET /scan?kinds=url,ip&skill=<name> read-only aggregated scan (one shot)
GET /scan/stream?kinds=...&skill=... scan with per-skill NDJSON progress
POST /preview dry-run diff of a replacement set
POST /apply write changes (one shot, with backup)
POST /apply/stream write changes with per-file progress
GET /backups list restorable backups (newest first)
POST /rollback/{backup_id} restore a backup
"""
from __future__ import annotations
import asyncio
import json
import logging
import shutil
from collections.abc import AsyncIterator
from datetime import UTC, datetime
from pathlib import Path
from fastapi import APIRouter, HTTPException, Query, Request
from fastapi.responses import StreamingResponse
from pydantic import BaseModel, ConfigDict, Field
from app.gateway.deps import get_optional_user_from_request
from deerflow.agents.lead_agent.prompt import refresh_skills_system_prompt_cache_async
from deerflow.skills.address_scan import (
VALID_KINDS,
apply_replacements,
infer_kind,
is_valid_replacement,
normalize_kinds,
scan_occurrences,
)
from deerflow.skills.storage import get_or_new_skill_storage
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/skill-addresses", tags=["skill-addresses"])
_BACKUP_DIRNAME = ".address-edits"
_SCAN_SUBDIRS = ("public", "custom")
_SCAN_EXTS = (".md", ".py")
# ── schemas ────────────────────────────────────────────────────────────────
class OccurrenceModel(BaseModel):
skill: str
category: str # "public" | "custom"
file: str # POSIX path relative to the skills root
line: int
col: int
snippet: str
class AddressGroup(BaseModel):
address: str
kind: str # "url" | "ip"
count: int
occurrences: list[OccurrenceModel]
class ScanResponse(BaseModel):
groups: list[AddressGroup]
scanned_files: int
scanned_skills: int
kinds: list[str]
scanned_at: str
class Replacement(BaseModel):
model_config = ConfigDict(populate_by_name=True)
from_: str = Field(alias="from", min_length=1)
to: str = Field(min_length=1)
kind: str | None = None
class PreviewRequest(BaseModel):
replacements: list[Replacement] = Field(default_factory=list)
class PreviewFileHit(BaseModel):
file: str
line: int
before: str
after: str
class PreviewItem(BaseModel):
model_config = ConfigDict(populate_by_name=True)
from_: str = Field(serialization_alias="from")
to: str
kind: str
count: int
files: list[PreviewFileHit]
class PreviewResponse(BaseModel):
items: list[PreviewItem]
total: int
warnings: list[str]
expected_total: int
class ApplyRequest(BaseModel):
replacements: list[Replacement] = Field(default_factory=list)
expected_total: int | None = None
backup: bool = True
class ChangedFile(BaseModel):
file: str
count: int
class ApplyResponse(BaseModel):
changed_files: list[ChangedFile]
replaced_total: int
backup_id: str | None
warnings: list[str]
class ReplacementSummary(BaseModel):
model_config = ConfigDict(populate_by_name=True)
from_: str = Field(serialization_alias="from")
to: str
kind: str
class BackupEntry(BaseModel):
backup_id: str
files: int
created_at: str
replaced_total: int = 0
replacements: list[ReplacementSummary] = Field(default_factory=list)
class BackupListResponse(BaseModel):
backups: list[BackupEntry]
class RollbackResponse(BaseModel):
backup_id: str
restored_files: list[str]
# ── auth / path helpers ──────────────────────────────────────────────────────
async def _require_admin(request: Request) -> None:
"""Admin-only. Lets the call through when auth is disabled (no user)."""
user = await get_optional_user_from_request(request)
if user is None:
return
if getattr(user, "system_role", None) != "admin":
raise HTTPException(status_code=403, detail="技能地址管理仅限管理员")
def _skills_root() -> Path:
return Path(get_or_new_skill_storage().get_skills_root_path()).resolve()
def _is_within(root: Path, p: Path) -> bool:
try:
p.resolve().relative_to(root)
return True
except (ValueError, OSError):
return False
def _skill_of(root: Path, path: Path) -> tuple[str, str, str]:
"""Return ``(category, skill_name, posix_rel_path)`` for a scanned file."""
rel = path.relative_to(root)
parts = rel.parts
category = parts[0] if parts else ""
skill_name = parts[1] if len(parts) > 1 else ""
return category, skill_name, rel.as_posix()
def _read_text(path: Path) -> str | None:
"""Read UTF-8 text preserving original line endings; skip binary/bad files."""
try:
with open(path, "r", encoding="utf-8", newline="") as fh:
return fh.read()
except (UnicodeDecodeError, OSError):
return None
def _write_text(path: Path, text: str) -> None:
with open(path, "w", encoding="utf-8", newline="") as fh:
fh.write(text)
# ── synchronous workers (run via asyncio.to_thread; never block the loop) ─────
def _iter_skill_files(root: Path, skill: str | None = None) -> list[Path]:
"""All ``*.md`` / ``*.py`` under ``{root}/{public,custom}``.
Hidden directories (``.address-edits`` backups, ``.history``) are skipped.
When *skill* is given, only that skill's directory is walked.
"""
files: list[Path] = []
for sub in _SCAN_SUBDIRS:
base = root / sub
if skill:
base = base / skill
if not base.is_dir():
continue
for p in base.rglob("*"):
if not p.is_file() or p.suffix.lower() not in _SCAN_EXTS:
continue
try:
rel_parts = p.relative_to(root).parts
except ValueError:
continue
if any(part.startswith(".") for part in rel_parts):
continue
files.append(p)
return files
def _group_files_by_skill(root: Path, files: list[Path]) -> dict[tuple[str, str], list[Path]]:
"""Bucket scanned files under their owning ``(category, skill_name)``."""
buckets: dict[tuple[str, str], list[Path]] = {}
for p in files:
category, skill_name, _ = _skill_of(root, p)
buckets.setdefault((category, skill_name), []).append(p)
return buckets
def _occurrence_kind(value: str) -> str:
return infer_kind(value) or "url"
def _merge_occurrences(groups: dict[str, AddressGroup], pairs: list[tuple[str, OccurrenceModel]]) -> None:
"""Merge ``(address, occurrence)`` pairs into the aggregate group map."""
for address, occ in pairs:
g = groups.get(address)
if g is None:
g = AddressGroup(address=address, kind=_occurrence_kind(address), count=0, occurrences=[])
groups[address] = g
g.count += 1
g.occurrences.append(occ)
def _plan_apply(root: Path, mapping: dict[str, str]) -> tuple[list[tuple[Path, str, int, str]], int]:
"""Compute the rewrite plan without touching disk → (plan, current_total)."""
plan: list[tuple[Path, str, int, str]] = []
current_total = 0
for path in _iter_skill_files(root):
text = _read_text(path)
if text is None:
continue
new_text, cnt = apply_replacements(text, mapping, VALID_KINDS)
if cnt <= 0:
continue
if not _is_within(root, path):
raise HTTPException(status_code=400, detail=f"拒绝写入技能目录之外的文件:{path}")
_, _, rel = _skill_of(root, path)
plan.append((path, new_text, cnt, rel))
current_total += cnt
return plan, current_total
def _backup_file(root: Path, backup_id: str, path: Path, rel: str) -> None:
dest = root / _BACKUP_DIRNAME / backup_id / rel
dest.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(path, dest)
_MANIFEST_SUFFIX = ".manifest.json"
def _manifest_path(root: Path, backup_id: str) -> Path:
"""Manifest lives as a *sibling* of the backup dir (``{id}.manifest.json``),
not inside it — so it is neither restored back into the skills tree on
rollback nor counted as a backed-up file."""
return root / _BACKUP_DIRNAME / f"{backup_id}{_MANIFEST_SUFFIX}"
def _write_manifest(
root: Path,
backup_id: str,
norm: list[tuple[str, str, str]],
changed: list["ChangedFile"],
replaced_total: int,
created_at: str,
) -> None:
"""Record what this edit changed so the UI can show a meaningful, per-edit
history (原地址 → 新地址, 文件数, 处数) and roll a single record back."""
data = {
"created_at": created_at,
"replaced_total": replaced_total,
"replacements": [{"from": frm, "to": to, "kind": kind} for frm, to, kind in norm],
"changed_files": [{"file": c.file, "count": c.count} for c in changed],
}
try: # best-effort: a metadata write failure must not fail the apply itself
path = _manifest_path(root, backup_id)
path.parent.mkdir(parents=True, exist_ok=True)
with open(path, "w", encoding="utf-8") as fh:
json.dump(data, fh, ensure_ascii=False)
except OSError:
logger.warning("Failed to write skill-address backup manifest for %s", backup_id, exc_info=True)
def _read_manifest(root: Path, backup_id: str) -> dict | None:
try:
with open(_manifest_path(root, backup_id), "r", encoding="utf-8") as fh:
return json.load(fh)
except (OSError, json.JSONDecodeError):
return None
# ── scan extraction shared by one-shot + stream ───────────────────────────────
def _scan_files_with_addresses(root: Path, files: list[Path], wanted: tuple[str, ...]) -> tuple[list[tuple[str, OccurrenceModel]], int]:
"""Scan files → ([(address, occurrence)], files_read)."""
pairs: list[tuple[str, OccurrenceModel]] = []
read = 0
for path in files:
text = _read_text(path)
if text is None:
continue
read += 1
category, skill_name, rel = _skill_of(root, path)
for o in scan_occurrences(text, wanted):
pairs.append(
(
o.value,
OccurrenceModel(skill=skill_name, category=category, file=rel, line=o.line, col=o.col, snippet=o.snippet),
)
)
return pairs, read
def _ordered_groups(groups: dict[str, AddressGroup]) -> list[AddressGroup]:
return sorted(groups.values(), key=lambda g: (-g.count, g.address))
def _build_mapping(replacements: list[Replacement]) -> tuple[dict[str, str], list[tuple[str, str, str]], list[str]]:
"""Validate replacements → ``(mapping, [(from,to,kind)], warnings)``.
Drops (with a warning) any entry that is empty, duplicated, not a recognized
URL/IP, a no-op (``from == to``), or whose ``to`` is malformed for its kind.
"""
mapping: dict[str, str] = {}
norm: list[tuple[str, str, str]] = []
warnings: list[str] = []
seen: set[str] = set()
for r in replacements:
frm = r.from_.strip()
to = r.to.strip()
if not frm:
continue
if frm in seen:
warnings.append(f"重复的原始地址已忽略:{frm}")
continue
kind = (r.kind or "").strip().lower() or infer_kind(frm)
if kind not in VALID_KINDS:
warnings.append(f"无法识别为 URL/IP,已忽略:{frm}")
continue
if to == frm:
warnings.append(f"替换值与原值相同,已忽略:{frm}")
continue
if not is_valid_replacement(kind, to):
warnings.append(f"替换值不是合法的 {kind},已忽略:{to}")
continue
seen.add(frm)
mapping[frm] = to
norm.append((frm, to, kind))
return mapping, norm, warnings
def _ndjson(obj: dict) -> bytes:
return (json.dumps(obj, ensure_ascii=False) + "\n").encode("utf-8")
# ── routes: scan ─────────────────────────────────────────────────────────────
@router.get("/scan", response_model=ScanResponse)
async def scan_addresses(
request: Request,
kinds: str | None = Query(default=None, description="逗号分隔:url,ip(默认全部)"),
skill: str | None = Query(default=None, description="仅扫描指定技能目录"),
) -> ScanResponse:
await _require_admin(request)
root = _skills_root()
wanted = normalize_kinds(kinds)
files = await asyncio.to_thread(_iter_skill_files, root, skill)
buckets = _group_files_by_skill(root, files)
groups: dict[str, AddressGroup] = {}
scanned = 0
for paths in buckets.values():
pairs, read = await asyncio.to_thread(_scan_files_with_addresses, root, paths, wanted)
_merge_occurrences(groups, pairs)
scanned += read
return ScanResponse(
groups=_ordered_groups(groups),
scanned_files=scanned,
scanned_skills=len(buckets),
kinds=list(wanted),
scanned_at=datetime.now(UTC).isoformat(),
)
@router.get("/scan/stream")
async def scan_addresses_stream(
request: Request,
kinds: str | None = Query(default=None),
skill: str | None = Query(default=None),
) -> StreamingResponse:
"""Scan with per-skill NDJSON progress.
Emits ``{"type":"start","total":N}`` then one
``{"type":"progress","done":i,"total":N,"skill":name}`` per skill, finally
``{"type":"result", ...ScanResponse...}``.
"""
await _require_admin(request)
root = _skills_root()
wanted = normalize_kinds(kinds)
async def gen() -> AsyncIterator[bytes]:
files = await asyncio.to_thread(_iter_skill_files, root, skill)
buckets = _group_files_by_skill(root, files)
total = len(buckets)
yield _ndjson({"type": "start", "total": total})
groups: dict[str, AddressGroup] = {}
scanned = 0
done = 0
for (category, skill_name), paths in buckets.items():
pairs, read = await asyncio.to_thread(_scan_files_with_addresses, root, paths, wanted)
_merge_occurrences(groups, pairs)
scanned += read
done += 1
yield _ndjson({"type": "progress", "done": done, "total": total, "skill": skill_name or category})
result = ScanResponse(
groups=_ordered_groups(groups),
scanned_files=scanned,
scanned_skills=total,
kinds=list(wanted),
scanned_at=datetime.now(UTC).isoformat(),
)
yield _ndjson({"type": "result", **result.model_dump()})
return StreamingResponse(gen(), media_type="application/x-ndjson")
# ── routes: preview ──────────────────────────────────────────────────────────
def _compute_preview(root: Path, mapping: dict[str, str], norm: list[tuple[str, str, str]]) -> tuple[list[PreviewItem], int]:
counts: dict[str, int] = {frm: 0 for frm, _, _ in norm}
files: dict[str, list[PreviewFileHit]] = {frm: [] for frm, _, _ in norm}
total = 0
if mapping:
for path in _iter_skill_files(root):
text = _read_text(path)
if text is None:
continue
_, _, rel = _skill_of(root, path)
for o in scan_occurrences(text, VALID_KINDS):
if o.value not in mapping:
continue
to = mapping[o.value]
counts[o.value] += 1
total += 1
files[o.value].append(PreviewFileHit(file=rel, line=o.line, before=o.snippet, after=o.snippet.replace(o.value, to)))
items = [PreviewItem(from_=frm, to=to, kind=kind, count=counts[frm], files=files[frm]) for frm, to, kind in norm]
return items, total
@router.post("/preview", response_model=PreviewResponse)
async def preview_replacements(request: Request, body: PreviewRequest) -> PreviewResponse:
await _require_admin(request)
root = _skills_root()
mapping, norm, warnings = _build_mapping(body.replacements)
items, total = await asyncio.to_thread(_compute_preview, root, mapping, norm)
return PreviewResponse(items=items, total=total, warnings=warnings, expected_total=total)
# ── routes: apply ────────────────────────────────────────────────────────────
def _new_backup_id() -> str:
return datetime.now(UTC).strftime("%Y%m%d-%H%M%S-%f")
@router.post("/apply", response_model=ApplyResponse)
async def apply_replacements_route(request: Request, body: ApplyRequest) -> ApplyResponse:
await _require_admin(request)
root = _skills_root()
mapping, norm, warnings = _build_mapping(body.replacements)
if not mapping:
return ApplyResponse(changed_files=[], replaced_total=0, backup_id=None, warnings=warnings)
plan, current_total = await asyncio.to_thread(_plan_apply, root, mapping)
if body.expected_total is not None and body.expected_total != current_total:
raise HTTPException(
status_code=409,
detail=f"文件自上次扫描后已变化(预期 {body.expected_total} 处,现为 {current_total} 处),请重新扫描后再应用。",
)
if not plan:
return ApplyResponse(changed_files=[], replaced_total=0, backup_id=None, warnings=warnings)
backup_id: str | None = None
if body.backup:
backup_id = _new_backup_id()
for path, _new, _cnt, rel in plan:
await asyncio.to_thread(_backup_file, root, backup_id, path, rel)
changed: list[ChangedFile] = []
replaced_total = 0
for path, new_text, cnt, rel in plan:
await asyncio.to_thread(_write_text, path, new_text)
changed.append(ChangedFile(file=rel, count=cnt))
replaced_total += cnt
if backup_id:
await asyncio.to_thread(_write_manifest, root, backup_id, norm, changed, replaced_total, datetime.now(UTC).isoformat())
await _refresh_cache_safe()
logger.info("Skill address apply: %d files, %d replacements, backup=%s", len(changed), replaced_total, backup_id)
return ApplyResponse(changed_files=changed, replaced_total=replaced_total, backup_id=backup_id, warnings=warnings)
@router.post("/apply/stream")
async def apply_replacements_stream(request: Request, body: ApplyRequest) -> StreamingResponse:
"""Apply with per-file NDJSON progress.
Validation + the optimistic-lock check happen *before* streaming (so a 409
is a normal HTTP error). Then: ``{"type":"start","total_files":M,...}`` →
one ``{"type":"progress","done":k,...}`` per written file →
``{"type":"result", ...ApplyResponse...}``.
"""
await _require_admin(request)
root = _skills_root()
mapping, norm, warnings = _build_mapping(body.replacements)
plan: list[tuple[Path, str, int, str]] = []
current_total = 0
if mapping:
plan, current_total = await asyncio.to_thread(_plan_apply, root, mapping)
if body.expected_total is not None and body.expected_total != current_total:
raise HTTPException(
status_code=409,
detail=f"文件自上次扫描后已变化(预期 {body.expected_total} 处,现为 {current_total} 处),请重新扫描后再应用。",
)
async def gen() -> AsyncIterator[bytes]:
if not plan:
result = ApplyResponse(changed_files=[], replaced_total=0, backup_id=None, warnings=warnings)
yield _ndjson({"type": "result", **result.model_dump()})
return
backup_id: str | None = None
if body.backup:
backup_id = _new_backup_id()
yield _ndjson({"type": "start", "total_files": len(plan), "total_replacements": current_total})
changed: list[ChangedFile] = []
replaced_total = 0
done = 0
for path, new_text, cnt, rel in plan:
if backup_id:
await asyncio.to_thread(_backup_file, root, backup_id, path, rel)
await asyncio.to_thread(_write_text, path, new_text)
changed.append(ChangedFile(file=rel, count=cnt))
replaced_total += cnt
done += 1
yield _ndjson({"type": "progress", "done": done, "total_files": len(plan), "file": rel, "count": cnt})
if backup_id:
await asyncio.to_thread(_write_manifest, root, backup_id, norm, changed, replaced_total, datetime.now(UTC).isoformat())
await _refresh_cache_safe()
logger.info("Skill address apply(stream): %d files, %d replacements, backup=%s", len(changed), replaced_total, backup_id)
result = ApplyResponse(changed_files=changed, replaced_total=replaced_total, backup_id=backup_id, warnings=warnings)
yield _ndjson({"type": "result", **result.model_dump()})
return StreamingResponse(gen(), media_type="application/x-ndjson")
# ── routes: backups / rollback ───────────────────────────────────────────────
def _list_backups(root: Path) -> list[BackupEntry]:
backup_root = root / _BACKUP_DIRNAME
entries: list[BackupEntry] = []
if backup_root.is_dir():
for d in backup_root.iterdir():
if not d.is_dir():
continue
file_count = sum(1 for p in d.rglob("*") if p.is_file())
manifest = _read_manifest(root, d.name)
created = ""
replaced_total = 0
replacements: list[ReplacementSummary] = []
if manifest:
created = str(manifest.get("created_at") or "")
replaced_total = int(manifest.get("replaced_total") or 0)
replacements = [
ReplacementSummary(from_=str(r.get("from", "")), to=str(r.get("to", "")), kind=str(r.get("kind", "")))
for r in manifest.get("replacements", [])
if isinstance(r, dict)
]
if not created:
try:
created = datetime.fromtimestamp(d.stat().st_mtime, UTC).isoformat()
except OSError:
created = ""
entries.append(
BackupEntry(
backup_id=d.name,
files=file_count,
created_at=created,
replaced_total=replaced_total,
replacements=replacements,
)
)
entries.sort(key=lambda e: e.backup_id, reverse=True)
return entries
@router.get("/backups", response_model=BackupListResponse)
async def list_backups(request: Request) -> BackupListResponse:
await _require_admin(request)
root = _skills_root()
entries = await asyncio.to_thread(_list_backups, root)
return BackupListResponse(backups=entries)
def _restore_backup(root: Path, backup_root: Path) -> list[str]:
restored: list[str] = []
for src in backup_root.rglob("*"):
if not src.is_file():
continue
rel = src.relative_to(backup_root)
dest = root / rel
if not _is_within(root, dest):
continue
dest.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(src, dest)
restored.append(rel.as_posix())
return restored
@router.post("/rollback/{backup_id}", response_model=RollbackResponse)
async def rollback(request: Request, backup_id: str) -> RollbackResponse:
await _require_admin(request)
root = _skills_root()
if "/" in backup_id or "\\" in backup_id or backup_id in (".", ".."):
raise HTTPException(status_code=400, detail="非法的备份编号")
backup_root = (root / _BACKUP_DIRNAME / backup_id).resolve()
if not _is_within(root, backup_root) or not backup_root.is_dir():
raise HTTPException(status_code=404, detail="备份不存在")
restored = await asyncio.to_thread(_restore_backup, root, backup_root)
await _refresh_cache_safe()
logger.info("Skill address rollback: backup=%s, restored %d files", backup_id, len(restored))
return RollbackResponse(backup_id=backup_id, restored_files=restored)
# ── misc ─────────────────────────────────────────────────────────────────────
async def _refresh_cache_safe() -> None:
try:
await refresh_skills_system_prompt_cache_async()
except Exception: # noqa: BLE001 — cache refresh is best-effort
logger.warning("Failed to refresh skills prompt cache", exc_info=True)
@@ -222,6 +222,10 @@ _CONTEXT_CONFIGURABLE_KEYS: frozenset[str] = frozenset(
"model_name",
"mode",
"thinking_enabled",
# 强制关闭思考(内网 vLLM/Qwen 模型只声明 supports_thinking 时,单靠
# thinking_enabled=False 不会注入 enable_thinking=false)。透传到 run config
# 后由 lead agent 交给 create_chat_model(force_disable_thinking=...)。
"thinking_force_disabled",
"reasoning_effort",
"rag_mode_enabled",
"writing_mode",