Merge branch 'ji' into master
合并 ji 的功能:开放问答接口(/api/open/chat[/stream] 回传生成文件 + 临时不留存)、 技能地址管理、智能体详情「最近对话」等。 冲突 frontend StrategyRoutes.tsx 已解决:同时保留 admin/roundtable-diagnostics(master) 与 admin/skill-addresses(ji) 两个管理员路由。 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
commit
fbec222eb4
38 files changed
+3365
-94
No files matched your search
@@ -50,6 +50,7 @@ from app.gateway.routers import (
|
||||
models,
|
||||
multi_agent,
|
||||
notifications,
|
||||
open_chat,
|
||||
parallel_agent_tasks,
|
||||
parallel_agents,
|
||||
positions,
|
||||
@@ -69,6 +70,7 @@ from app.gateway.routers import (
|
||||
runs,
|
||||
scheduled_tasks,
|
||||
sensitive_words,
|
||||
skill_addresses,
|
||||
skills,
|
||||
suggestions,
|
||||
system_settings,
|
||||
@@ -1359,6 +1361,9 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
# Sensitive-word map (敏感词管理 / 脱敏词) at /api/sensitive-words
|
||||
app.include_router(sensitive_words.router)
|
||||
|
||||
# Skill URL/IP batch replace (技能地址管理) at /api/skill-addresses
|
||||
app.include_router(skill_addresses.router)
|
||||
|
||||
# 大屏绘制智能体独立页面会话(按 taskId 存取,不分权)at /api/dashboard-sessions
|
||||
app.include_router(dashboard_sessions.router)
|
||||
|
||||
@@ -1456,6 +1461,9 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
# Stateless Runs API (stream/wait without a pre-existing thread)
|
||||
app.include_router(runs.router)
|
||||
|
||||
# 开放问答接口 /api/open/chat —— 无需登录,指定模型 + 智能体做问答,可关闭思考模式
|
||||
app.include_router(open_chat.router)
|
||||
|
||||
# --- ?????????? /page/workspace/roundtable/planning?---
|
||||
# ??? app/gateway/routers/{multi_agent,intent,recommend}.py?
|
||||
# ??? frontend-web/docs/multi-agent-backend-dev.md
|
||||
|
||||
@@ -35,6 +35,9 @@ _PUBLIC_PATH_PREFIXES: tuple[str, ...] = (
|
||||
"/redoc",
|
||||
"/openapi.json",
|
||||
"/api/public/",
|
||||
# 开放问答接口:无需登录即可指定模型 + 智能体做问答(见 routers/open_chat.py)。
|
||||
# 以 "default" 用户桶运行,只暴露受控的问答能力。
|
||||
"/api/open/",
|
||||
)
|
||||
|
||||
# Exact auth paths that are public (login/register/status check).
|
||||
|
||||
@@ -52,6 +52,10 @@ def should_check_csrf(request: Request) -> bool:
|
||||
# PUT/POST returns 403 even though auth was bypassed earlier in the chain.
|
||||
if path.startswith("/api/public/"):
|
||||
return False
|
||||
# 开放问答接口同样是匿名、无会话的(见 auth_middleware._PUBLIC_PATH_PREFIXES),
|
||||
# 双提交 Cookie 既无 token 可比、也无会话可被伪造,豁免之。
|
||||
if path.startswith("/api/open/"):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,432 @@
|
||||
"""开放问答接口(无需登录)。
|
||||
|
||||
给外部系统一个最简单的 HTTP 入口来「调用本项目的智能体做问答」:
|
||||
|
||||
- 可指定 **模型**(``model_name``,对应 ``config.yaml`` 中 ``models[]`` 的 name)。
|
||||
- 可指定 **智能体**(``agent_id``,对应 ``.deer-flow/agents/<id>/``;留空 = 默认 lead agent)。
|
||||
- 可关闭 **大模型思考模式**(``thinking_enabled=false``,默认即关闭)。内网 vLLM/Qwen
|
||||
模型仅靠 ``thinking_enabled=False`` 往往关不掉思考(其 config 只声明
|
||||
``supports_thinking: true``),本接口在关闭思考时会把
|
||||
``thinking_force_disabled`` 一并下发,经
|
||||
``create_chat_model(force_disable_thinking=True)`` 强制注入
|
||||
``extra_body.chat_template_kwargs.enable_thinking=false``(及 GLM 的
|
||||
``thinking.type=disabled``)——与「生成标题」时关闭思考的做法一致。
|
||||
|
||||
该路由前缀 ``/api/open/`` 已在 ``auth_middleware`` 白名单里,**绕过登录鉴权**,
|
||||
以 ``"default"`` 用户桶运行,只暴露受控的问答能力(无文件/线程持久化)。
|
||||
|
||||
请求示例::
|
||||
|
||||
POST /api/open/chat
|
||||
{
|
||||
"message": "用一句话介绍一下你自己",
|
||||
"model_name": "deepseek-ai/DeepSeek-V4-Flash",
|
||||
"agent_id": null,
|
||||
"thinking_enabled": false
|
||||
}
|
||||
|
||||
返回::
|
||||
|
||||
{"answer": "...", "model_name": "...", "agent_id": null,
|
||||
"thinking_enabled": false, "thread_id": "open-...."}
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import shutil
|
||||
import uuid
|
||||
from collections.abc import AsyncIterator
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from fastapi.responses import StreamingResponse
|
||||
from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, SystemMessage
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from deerflow.agents.lead_agent.agent import make_lead_agent
|
||||
from deerflow.config.agents_config import list_custom_agents, validate_agent_id
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/open", tags=["open"])
|
||||
|
||||
_THINK_TAG_RE = re.compile(r"<think>[\s\S]*?</think>", re.IGNORECASE)
|
||||
|
||||
|
||||
class OpenChatMessage(BaseModel):
|
||||
"""多轮历史中的一条消息。"""
|
||||
|
||||
role: Literal["user", "assistant", "system"] = "user"
|
||||
content: str
|
||||
|
||||
|
||||
class OpenChatRequest(BaseModel):
|
||||
"""开放问答请求体。
|
||||
|
||||
``message`` 与 ``messages`` 二选一:单轮直接传 ``message``;需要带上下文时传
|
||||
``messages``(完整历史,本接口本身无状态、不落库)。
|
||||
"""
|
||||
|
||||
message: str | None = Field(default=None, description="单轮问题文本;与 messages 二选一")
|
||||
messages: list[OpenChatMessage] | None = Field(default=None, description="多轮历史;与 message 二选一")
|
||||
model_name: str | None = Field(default=None, description="models[] 中的模型名;留空用默认模型")
|
||||
agent_name: str | None = Field(default=None, description="智能体名称(中文显示名);留空用默认 lead agent")
|
||||
agent_id: str | None = Field(default=None, description="智能体 id;一般不用,优先级高于 agent_name")
|
||||
thinking_enabled: bool = Field(default=False, description="是否开启思考;默认 False = 关闭思考(内网模型也会被强制关闭)")
|
||||
strip_think: bool = Field(default=True, description="关闭思考时,从答案里剥离残留的 <think>…</think> 块")
|
||||
thread_id: str | None = Field(default=None, description="可选;仅用于文件隔离,留空自动生成")
|
||||
recursion_limit: int = Field(default=100, ge=1, le=300, description="智能体单次问答的最大步数")
|
||||
include_files: bool = Field(default=True, description="是否随结果回传智能体在 outputs 目录生成的文件(名称+内容)")
|
||||
|
||||
|
||||
class OpenChatFile(BaseModel):
|
||||
"""智能体本次运行在 /mnt/user-data/outputs 下生成的一个文件。"""
|
||||
|
||||
name: str # outputs/ 下的相对路径,如 "报告.md"
|
||||
virtual_path: str # 沙箱虚拟路径,如 "/mnt/user-data/outputs/报告.md"
|
||||
encoding: Literal["text", "base64"] = "text" # text=可直接用的文本;base64=二进制需解码
|
||||
content: str
|
||||
|
||||
|
||||
class OpenChatResponse(BaseModel):
|
||||
answer: str
|
||||
model_name: str | None
|
||||
agent_name: str | None
|
||||
agent_id: str | None
|
||||
thinking_enabled: bool
|
||||
thread_id: str
|
||||
files: list[OpenChatFile] = Field(default_factory=list, description="本次生成的输出文件(含内容)")
|
||||
|
||||
|
||||
def _resolve_agent_id(agent_id: str | None, agent_name: str | None) -> str | None:
|
||||
"""把 agent_id / 中文名称 解析成运行 id。
|
||||
|
||||
优先级:显式 agent_id > agent_name(按 .deer-flow/agents/*/config.yaml 的 name 匹配)。
|
||||
都没传 = None(默认 lead agent)。名称匹配先精确、再忽略大小写/首尾空格。
|
||||
"""
|
||||
if agent_id and agent_id.strip():
|
||||
try:
|
||||
return validate_agent_id(agent_id)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=422, detail=f"非法的 agent_id:{exc}") from exc
|
||||
|
||||
if not agent_name or not agent_name.strip():
|
||||
return None
|
||||
|
||||
target = agent_name.strip()
|
||||
agents = list_custom_agents()
|
||||
for a in agents:
|
||||
if (a.name or "").strip() == target:
|
||||
return a.id
|
||||
lowered = target.lower()
|
||||
for a in agents:
|
||||
if (a.name or "").strip().lower() == lowered:
|
||||
return a.id
|
||||
raise HTTPException(status_code=404, detail=f"未找到名称为「{agent_name}」的智能体。")
|
||||
|
||||
|
||||
def _build_lc_messages(body: OpenChatRequest) -> list[BaseMessage]:
|
||||
"""把请求体转成 LangChain 消息列表。"""
|
||||
if body.messages:
|
||||
out: list[BaseMessage] = []
|
||||
for m in body.messages:
|
||||
text = (m.content or "").strip()
|
||||
if not text:
|
||||
continue
|
||||
if m.role == "assistant":
|
||||
out.append(AIMessage(content=m.content))
|
||||
elif m.role == "system":
|
||||
out.append(SystemMessage(content=m.content))
|
||||
else:
|
||||
out.append(HumanMessage(content=m.content))
|
||||
if out:
|
||||
return out
|
||||
if body.message and body.message.strip():
|
||||
return [HumanMessage(content=body.message)]
|
||||
raise HTTPException(status_code=422, detail="必须提供非空的 'message' 或 'messages'。")
|
||||
|
||||
|
||||
def _extract_text(content: Any) -> str:
|
||||
"""从 AIMessage.content(str 或 block 列表)抽取纯文本。"""
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
parts: list[str] = []
|
||||
for block in content:
|
||||
if isinstance(block, str):
|
||||
parts.append(block)
|
||||
elif isinstance(block, dict):
|
||||
text_value = block.get("text")
|
||||
if isinstance(text_value, str):
|
||||
parts.append(text_value)
|
||||
return "".join(parts) if parts else ""
|
||||
return ""
|
||||
|
||||
|
||||
def _extract_answer(result: Any, *, strip_think: bool) -> str:
|
||||
"""从 ainvoke 的最终状态里取最后一条 AI 文本。"""
|
||||
messages = (result or {}).get("messages") if isinstance(result, dict) else None
|
||||
answer = ""
|
||||
if isinstance(messages, list):
|
||||
for msg in reversed(messages):
|
||||
if isinstance(msg, AIMessage):
|
||||
text = _extract_text(msg.content).strip()
|
||||
if text:
|
||||
answer = text
|
||||
break
|
||||
if strip_think and answer:
|
||||
answer = _THINK_TAG_RE.sub("", answer).strip()
|
||||
return answer
|
||||
|
||||
|
||||
_OUTPUTS_VIRTUAL_PREFIX = "/mnt/user-data/outputs"
|
||||
_MAX_OUTPUT_FILES = 20
|
||||
_MAX_OUTPUT_FILE_BYTES = 5 * 1024 * 1024 # 单文件 5MB 上限,避免把大二进制塞进 JSON 响应
|
||||
|
||||
|
||||
def _collect_output_files(state: Any, *, include: bool) -> list[OpenChatFile]:
|
||||
"""读取本次运行在沙箱 outputs 目录生成的文件,随结果一并回传。
|
||||
|
||||
开放问答是无状态的(不持久化线程),客户端没法用 artifacts 路由按线程取件;但本端点
|
||||
就跑在服务端进程里,沙箱把生成的文件落在 ``state['thread_data']['outputs_path']``,
|
||||
所以这里直接扫描该目录,把文件内容带回(文本按 utf-8,二进制按 base64)。
|
||||
受 ``_MAX_OUTPUT_FILES`` / ``_MAX_OUTPUT_FILE_BYTES`` 限制。
|
||||
"""
|
||||
if not include or not isinstance(state, dict):
|
||||
return []
|
||||
thread_data = state.get("thread_data") or {}
|
||||
outputs_path = thread_data.get("outputs_path")
|
||||
if not outputs_path:
|
||||
return []
|
||||
base = Path(outputs_path)
|
||||
if not base.is_dir():
|
||||
return []
|
||||
try:
|
||||
entries = sorted((p for p in base.rglob("*") if p.is_file()), key=lambda p: p.as_posix())
|
||||
except OSError:
|
||||
return []
|
||||
|
||||
files: list[OpenChatFile] = []
|
||||
for entry in entries:
|
||||
if len(files) >= _MAX_OUTPUT_FILES:
|
||||
break
|
||||
try:
|
||||
if entry.stat().st_size > _MAX_OUTPUT_FILE_BYTES:
|
||||
continue
|
||||
data = entry.read_bytes()
|
||||
except OSError:
|
||||
continue
|
||||
rel = entry.relative_to(base).as_posix()
|
||||
try:
|
||||
content = data.decode("utf-8")
|
||||
encoding: Literal["text", "base64"] = "text"
|
||||
except UnicodeDecodeError:
|
||||
content = base64.b64encode(data).decode("ascii")
|
||||
encoding = "base64"
|
||||
files.append(OpenChatFile(name=rel, virtual_path=f"{_OUTPUTS_VIRTUAL_PREFIX}/{rel}", encoding=encoding, content=content))
|
||||
return files
|
||||
|
||||
|
||||
def _cleanup_ephemeral_thread(state: Any, thread_id: str | None) -> None:
|
||||
"""删除本次开放问答的临时线程目录(open-*),落实"聊天记录不存储"。
|
||||
|
||||
生成文件已在 _collect_output_files 读进内存(随响应回传),这里把磁盘上整个临时线程
|
||||
目录清掉,避免无状态调用在服务端堆积残留。仅对 ``open-*`` 线程生效,best-effort。
|
||||
"""
|
||||
if not thread_id or not str(thread_id).startswith("open-"):
|
||||
return
|
||||
thread_data = state.get("thread_data") if isinstance(state, dict) else None
|
||||
outputs_path = (thread_data or {}).get("outputs_path")
|
||||
if not outputs_path:
|
||||
return
|
||||
try:
|
||||
p = Path(outputs_path).resolve()
|
||||
# outputs_path 形如 .../threads/<thread_id>/user-data/outputs;只删到 <thread_id> 这层
|
||||
thread_root = next((parent for parent in p.parents if parent.name == thread_id), None)
|
||||
if thread_root is not None and thread_root.exists():
|
||||
shutil.rmtree(thread_root, ignore_errors=True)
|
||||
except Exception as exc: # noqa: BLE001 — 清理失败不影响返回结果
|
||||
logger.warning("清理临时线程目录失败 (%s): %s", thread_id, exc)
|
||||
|
||||
|
||||
# 流式响应每隔这么久没有真实事件就发一个 SSE 注释心跳,持续制造下行字节,
|
||||
# 让前置反向代理(nginx proxy_read_timeout 默认 60s 等)不会把长任务判定成空闲超时。
|
||||
_HEARTBEAT_SECONDS = 15.0
|
||||
|
||||
|
||||
def _build_run_context(body: OpenChatRequest) -> tuple[dict[str, Any], str | None, str]:
|
||||
"""把请求体翻译成 (config, agent_id, thread_id)。两个端点共用,确保关思考等行为一致。"""
|
||||
agent_id = _resolve_agent_id(body.agent_id, body.agent_name)
|
||||
thread_id = body.thread_id or f"open-{uuid.uuid4()}"
|
||||
configurable: dict[str, Any] = {
|
||||
"thread_id": thread_id,
|
||||
"model_name": body.model_name,
|
||||
"thinking_enabled": body.thinking_enabled,
|
||||
# 关闭思考时强制下发,确保内网模型也真的关掉(见模块 docstring)。
|
||||
"thinking_force_disabled": not body.thinking_enabled,
|
||||
# 一次性问答:跳过自动起标题(省一次 LLM 调用)、不挂定时任务工具。
|
||||
"is_scheduled_run": True,
|
||||
# 临时/无状态运行:跳过记忆与知识库的读写(不召回、不留存、不自动沉淀),
|
||||
# 做到"通过本接口的聊天记录不存储"(见 _build_middlewares 的 is_ephemeral)。
|
||||
"ephemeral": True,
|
||||
"memory_injection_enabled": False,
|
||||
}
|
||||
if agent_id:
|
||||
configurable["agent_id"] = agent_id
|
||||
config: dict[str, Any] = {"configurable": configurable, "recursion_limit": body.recursion_limit}
|
||||
return config, agent_id, thread_id
|
||||
|
||||
|
||||
def _sse(event: str | None, data: Any) -> str:
|
||||
"""序列化一条 SSE 消息。event=None 时只发 data 行。"""
|
||||
payload = json.dumps(data, ensure_ascii=False)
|
||||
head = f"event: {event}\n" if event else ""
|
||||
return f"{head}data: {payload}\n\n"
|
||||
|
||||
|
||||
@router.post("/chat", response_model=OpenChatResponse)
|
||||
async def open_chat(body: OpenChatRequest) -> OpenChatResponse:
|
||||
"""指定模型 + 智能体做一次问答(无状态、可关闭思考)。
|
||||
|
||||
同步阻塞版:整段调研跑完才返回。**长任务(联网调研)请改用 /chat/stream**——
|
||||
本端点期间不产生任何下行字节,容易被前置反代的 proxy_read_timeout 掐成 504。
|
||||
"""
|
||||
lc_messages = _build_lc_messages(body)
|
||||
config, agent_id, thread_id = _build_run_context(body)
|
||||
|
||||
try:
|
||||
agent = make_lead_agent(config)
|
||||
result = await agent.ainvoke({"messages": lc_messages}, config=config)
|
||||
except Exception as exc: # noqa: BLE001 — 开放接口需把内部错误归一成 502
|
||||
logger.exception("open_chat 调用智能体失败 (model=%s, agent=%s)", body.model_name, agent_id)
|
||||
raise HTTPException(status_code=502, detail=f"智能体调用失败:{exc}") from exc
|
||||
|
||||
answer = _extract_answer(result, strip_think=body.strip_think and not body.thinking_enabled)
|
||||
files = _collect_output_files(result, include=body.include_files)
|
||||
_cleanup_ephemeral_thread(result, thread_id) # 取完文件即清理临时目录,不在服务端留存
|
||||
if not answer and not files:
|
||||
raise HTTPException(status_code=502, detail="智能体未返回任何文本答案或文件。")
|
||||
|
||||
return OpenChatResponse(
|
||||
answer=answer,
|
||||
model_name=body.model_name,
|
||||
agent_name=body.agent_name,
|
||||
agent_id=agent_id,
|
||||
thinking_enabled=body.thinking_enabled,
|
||||
thread_id=thread_id,
|
||||
files=files,
|
||||
)
|
||||
|
||||
|
||||
async def _open_chat_event_stream(body: OpenChatRequest) -> AsyncIterator[str]:
|
||||
"""驱动智能体并产出 SSE 事件流。
|
||||
|
||||
- 边跑边转发 AI 文本增量(``event: delta``,仅供实时观看,客户端可忽略);
|
||||
- 没有真实事件时每 ``_HEARTBEAT_SECONDS`` 发一个心跳注释行保活;
|
||||
- 全部跑完后用 ``event: result`` 给出最终(已按需剥离 <think> 的)答案;
|
||||
- 出错则 ``event: error``。
|
||||
|
||||
用「后台生产任务 + 队列 + 超时取队列」的方式把「智能体进度」与「定时心跳」解耦,
|
||||
所以即便某次工具调用静默很久,心跳仍照常下行,连接不会被反代判空闲。
|
||||
"""
|
||||
lc_messages = _build_lc_messages(body)
|
||||
config, agent_id, thread_id = _build_run_context(body)
|
||||
strip_think = body.strip_think and not body.thinking_enabled
|
||||
|
||||
queue: asyncio.Queue[tuple[str, Any]] = asyncio.Queue()
|
||||
final_state: dict[str, Any] = {}
|
||||
|
||||
async def produce() -> None:
|
||||
nonlocal final_state
|
||||
try:
|
||||
agent = make_lead_agent(config)
|
||||
async for mode, chunk in agent.astream({"messages": lc_messages}, config=config, stream_mode=["values", "messages"]):
|
||||
if mode == "values" and isinstance(chunk, dict):
|
||||
final_state = chunk
|
||||
await queue.put(("progress", {"messages": len(chunk.get("messages") or [])}))
|
||||
elif mode == "messages":
|
||||
msg = chunk[0] if isinstance(chunk, tuple) and chunk else chunk
|
||||
# 仅转发 AI 正文增量;DeepSeek 等把思考放在 additional_kwargs.reasoning_content,
|
||||
# 不在 content 里,所以这里天然不会泄漏 <think>。
|
||||
if isinstance(msg, AIMessage):
|
||||
text = _extract_text(getattr(msg, "content", "")).strip()
|
||||
if text:
|
||||
await queue.put(("delta", {"text": text}))
|
||||
except Exception as exc: # noqa: BLE001 — 开放接口需把内部错误归一成 error 事件
|
||||
logger.exception("open_chat_stream 调用智能体失败 (model=%s, agent=%s)", body.model_name, agent_id)
|
||||
await queue.put(("error", {"detail": f"智能体调用失败:{exc}"}))
|
||||
finally:
|
||||
await queue.put(("__end__", None))
|
||||
|
||||
task = asyncio.create_task(produce())
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
event, data = await asyncio.wait_for(queue.get(), timeout=_HEARTBEAT_SECONDS)
|
||||
except asyncio.TimeoutError:
|
||||
yield ": keepalive\n\n" # SSE 注释行,客户端忽略,仅用于保活
|
||||
continue
|
||||
if event == "__end__":
|
||||
break
|
||||
if event == "error":
|
||||
yield _sse("error", data)
|
||||
return
|
||||
yield _sse(event, data)
|
||||
|
||||
answer = _extract_answer(final_state, strip_think=strip_think)
|
||||
files = _collect_output_files(final_state, include=body.include_files)
|
||||
_cleanup_ephemeral_thread(final_state, thread_id) # 取完文件即清理临时目录,不在服务端留存
|
||||
if not answer and not files:
|
||||
yield _sse("error", {"detail": "智能体未返回任何文本答案或文件。"})
|
||||
return
|
||||
yield _sse(
|
||||
"result",
|
||||
{
|
||||
"answer": answer,
|
||||
"model_name": body.model_name,
|
||||
"agent_name": body.agent_name,
|
||||
"agent_id": agent_id,
|
||||
"thinking_enabled": body.thinking_enabled,
|
||||
"thread_id": thread_id,
|
||||
"files": [f.model_dump() for f in files],
|
||||
},
|
||||
)
|
||||
finally:
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
|
||||
|
||||
@router.post("/chat/stream")
|
||||
async def open_chat_stream(body: OpenChatRequest) -> StreamingResponse:
|
||||
"""流式版 /chat:边跑边发心跳保活,跑完用 ``event: result`` 给最终答案。
|
||||
|
||||
解决两件事:(1) 长任务(联网调研、报告)同步阻塞会被前置反代 proxy_read_timeout
|
||||
掐成 504;流式持续有字节下行,代理不再判定空闲超时。(2) 客户端只需读到
|
||||
``event: result`` 解析 ``answer`` 即可,中间的 ``delta``/``progress``/心跳都可忽略。
|
||||
"""
|
||||
# 入参校验放在流外:非法请求直接抛 4xx,而不是开了流再报错。
|
||||
_build_lc_messages(body)
|
||||
return StreamingResponse(
|
||||
_open_chat_event_stream(body),
|
||||
media_type="text/event-stream",
|
||||
headers={
|
||||
"Cache-Control": "no-cache",
|
||||
"Connection": "keep-alive",
|
||||
# 关掉 nginx 对该响应的缓冲,确保 SSE/心跳实时下行(即便前置 nginx 默认开缓冲)。
|
||||
"X-Accel-Buffering": "no",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/agents")
|
||||
async def open_list_agents() -> dict[str, Any]:
|
||||
"""列出可用智能体的名称(方便知道 agent_name 该填什么)。"""
|
||||
agents = list_custom_agents()
|
||||
items = [{"name": a.name, "id": a.id} for a in agents if (a.name or "").strip()]
|
||||
return {"agents": items, "count": len(items)}
|
||||
@@ -67,24 +67,11 @@ async def list_recommended_questions(
|
||||
) -> RecommendedQuestionsListResponse:
|
||||
store = _get_store(request)
|
||||
rows = await store.list_all()
|
||||
if not all:
|
||||
# Filter by the caller's 岗位 (position) visibility. ``None`` =
|
||||
# unrestricted (no skill/question tag binding → show all default
|
||||
# questions); a set = show only the questions carrying the bound tags.
|
||||
sync = getattr(request.app.state, "position_sync", None)
|
||||
if sync is not None:
|
||||
try:
|
||||
visible = await sync.resolve_visible(_current_user_id(request), "recommended_question")
|
||||
except Exception:
|
||||
logger.debug("recommended-question visibility resolve failed", exc_info=True)
|
||||
visible = None
|
||||
if visible is not None:
|
||||
rows = [r for r in rows if r["id"] in visible]
|
||||
responses = [RecommendedQuestionResponse(**r) for r in rows]
|
||||
|
||||
# Bulk-attach the tags assigned to each question so the management list can
|
||||
# display them and filter by 标签. Best-effort: a missing/failing tag store
|
||||
# never breaks the question list.
|
||||
# display them and the 岗位 filter below can tell tagged from untagged.
|
||||
# Best-effort: a missing/failing tag store never breaks the question list.
|
||||
tag_store = getattr(request.app.state, "tag_store", None)
|
||||
if tag_store is not None and responses:
|
||||
try:
|
||||
@@ -95,6 +82,24 @@ async def list_recommended_questions(
|
||||
for resp in responses:
|
||||
resp.tags = assignments.get(resp.id, [])
|
||||
|
||||
if not all:
|
||||
# Filter by the caller's 岗位 (position) visibility:
|
||||
# * ``set`` — the position binds recommended_question 标签 → show only
|
||||
# the questions carrying those bound tags.
|
||||
# * ``None`` — 默认岗位 / 无标签绑定 → show only **untagged** questions.
|
||||
# 打标的题目只在绑定了对应标签的岗位展示,不再泄漏到默认岗位。
|
||||
sync = getattr(request.app.state, "position_sync", None)
|
||||
if sync is not None:
|
||||
try:
|
||||
visible = await sync.resolve_visible(_current_user_id(request), "recommended_question")
|
||||
except Exception:
|
||||
logger.debug("recommended-question visibility resolve failed", exc_info=True)
|
||||
visible = None
|
||||
if visible is not None:
|
||||
responses = [r for r in responses if r.id in visible]
|
||||
else:
|
||||
responses = [r for r in responses if not r.tags]
|
||||
|
||||
if tag_ids:
|
||||
wanted = set(tag_ids)
|
||||
responses = [r for r in responses if any(t.get("id") in wanted for t in r.tags)]
|
||||
|
||||
@@ -0,0 +1,713 @@
|
||||
"""技能地址(URL/IP)批量替换 API (技能地址管理).
|
||||
|
||||
Scans the **running** skills directory (``skills/{public,custom}``) — the files
|
||||
the agent actually reads — for ``http(s)://`` URLs and IPv4 (``ip[:port]``)
|
||||
addresses, aggregates each unique address with every place it appears (across
|
||||
``.md`` and ``.py``, across files), and lets an admin rewrite them in bulk.
|
||||
|
||||
Why this shape: the same address routinely lives in both a skill's ``SKILL.md``
|
||||
(documented endpoint) and its ``scripts/*.py`` (the actual call), so the natural
|
||||
unit is *"one address → replace all of its occurrences at once"*. The pure
|
||||
scan/replace logic lives in ``deerflow.skills.address_scan`` (harness layer,
|
||||
substring-safe span replacement); this router only walks the tree, enforces
|
||||
admin auth + path safety, backs up before writing, and refreshes the skills
|
||||
system-prompt cache so edits take effect immediately.
|
||||
|
||||
**Scale / availability (150+ skills):** every filesystem walk / read / write is
|
||||
offloaded to a worker thread via ``asyncio.to_thread`` so the event loop is
|
||||
never blocked while scanning or rewriting — the Gateway stays responsive. The
|
||||
streaming endpoints additionally emit NDJSON progress (``已处理 X / 共 N``) so the
|
||||
UI can render a progress bar for both the scan and the apply phases.
|
||||
|
||||
Routes (prefix ``/api/skill-addresses``, **admin only**):
|
||||
GET /scan?kinds=url,ip&skill=<name> read-only aggregated scan (one shot)
|
||||
GET /scan/stream?kinds=...&skill=... scan with per-skill NDJSON progress
|
||||
POST /preview dry-run diff of a replacement set
|
||||
POST /apply write changes (one shot, with backup)
|
||||
POST /apply/stream write changes with per-file progress
|
||||
GET /backups list restorable backups (newest first)
|
||||
POST /rollback/{backup_id} restore a backup
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import shutil
|
||||
from collections.abc import AsyncIterator
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query, Request
|
||||
from fastapi.responses import StreamingResponse
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from app.gateway.deps import get_optional_user_from_request
|
||||
from deerflow.agents.lead_agent.prompt import refresh_skills_system_prompt_cache_async
|
||||
from deerflow.skills.address_scan import (
|
||||
VALID_KINDS,
|
||||
apply_replacements,
|
||||
infer_kind,
|
||||
is_valid_replacement,
|
||||
normalize_kinds,
|
||||
scan_occurrences,
|
||||
)
|
||||
from deerflow.skills.storage import get_or_new_skill_storage
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/skill-addresses", tags=["skill-addresses"])
|
||||
|
||||
_BACKUP_DIRNAME = ".address-edits"
|
||||
_SCAN_SUBDIRS = ("public", "custom")
|
||||
_SCAN_EXTS = (".md", ".py")
|
||||
|
||||
|
||||
# ── schemas ────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class OccurrenceModel(BaseModel):
|
||||
skill: str
|
||||
category: str # "public" | "custom"
|
||||
file: str # POSIX path relative to the skills root
|
||||
line: int
|
||||
col: int
|
||||
snippet: str
|
||||
|
||||
|
||||
class AddressGroup(BaseModel):
|
||||
address: str
|
||||
kind: str # "url" | "ip"
|
||||
count: int
|
||||
occurrences: list[OccurrenceModel]
|
||||
|
||||
|
||||
class ScanResponse(BaseModel):
|
||||
groups: list[AddressGroup]
|
||||
scanned_files: int
|
||||
scanned_skills: int
|
||||
kinds: list[str]
|
||||
scanned_at: str
|
||||
|
||||
|
||||
class Replacement(BaseModel):
|
||||
model_config = ConfigDict(populate_by_name=True)
|
||||
|
||||
from_: str = Field(alias="from", min_length=1)
|
||||
to: str = Field(min_length=1)
|
||||
kind: str | None = None
|
||||
|
||||
|
||||
class PreviewRequest(BaseModel):
|
||||
replacements: list[Replacement] = Field(default_factory=list)
|
||||
|
||||
|
||||
class PreviewFileHit(BaseModel):
|
||||
file: str
|
||||
line: int
|
||||
before: str
|
||||
after: str
|
||||
|
||||
|
||||
class PreviewItem(BaseModel):
|
||||
model_config = ConfigDict(populate_by_name=True)
|
||||
|
||||
from_: str = Field(serialization_alias="from")
|
||||
to: str
|
||||
kind: str
|
||||
count: int
|
||||
files: list[PreviewFileHit]
|
||||
|
||||
|
||||
class PreviewResponse(BaseModel):
|
||||
items: list[PreviewItem]
|
||||
total: int
|
||||
warnings: list[str]
|
||||
expected_total: int
|
||||
|
||||
|
||||
class ApplyRequest(BaseModel):
|
||||
replacements: list[Replacement] = Field(default_factory=list)
|
||||
expected_total: int | None = None
|
||||
backup: bool = True
|
||||
|
||||
|
||||
class ChangedFile(BaseModel):
|
||||
file: str
|
||||
count: int
|
||||
|
||||
|
||||
class ApplyResponse(BaseModel):
|
||||
changed_files: list[ChangedFile]
|
||||
replaced_total: int
|
||||
backup_id: str | None
|
||||
warnings: list[str]
|
||||
|
||||
|
||||
class ReplacementSummary(BaseModel):
|
||||
model_config = ConfigDict(populate_by_name=True)
|
||||
|
||||
from_: str = Field(serialization_alias="from")
|
||||
to: str
|
||||
kind: str
|
||||
|
||||
|
||||
class BackupEntry(BaseModel):
|
||||
backup_id: str
|
||||
files: int
|
||||
created_at: str
|
||||
replaced_total: int = 0
|
||||
replacements: list[ReplacementSummary] = Field(default_factory=list)
|
||||
|
||||
|
||||
class BackupListResponse(BaseModel):
|
||||
backups: list[BackupEntry]
|
||||
|
||||
|
||||
class RollbackResponse(BaseModel):
|
||||
backup_id: str
|
||||
restored_files: list[str]
|
||||
|
||||
|
||||
# ── auth / path helpers ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def _require_admin(request: Request) -> None:
|
||||
"""Admin-only. Lets the call through when auth is disabled (no user)."""
|
||||
|
||||
user = await get_optional_user_from_request(request)
|
||||
if user is None:
|
||||
return
|
||||
if getattr(user, "system_role", None) != "admin":
|
||||
raise HTTPException(status_code=403, detail="技能地址管理仅限管理员")
|
||||
|
||||
|
||||
def _skills_root() -> Path:
|
||||
return Path(get_or_new_skill_storage().get_skills_root_path()).resolve()
|
||||
|
||||
|
||||
def _is_within(root: Path, p: Path) -> bool:
|
||||
try:
|
||||
p.resolve().relative_to(root)
|
||||
return True
|
||||
except (ValueError, OSError):
|
||||
return False
|
||||
|
||||
|
||||
def _skill_of(root: Path, path: Path) -> tuple[str, str, str]:
|
||||
"""Return ``(category, skill_name, posix_rel_path)`` for a scanned file."""
|
||||
|
||||
rel = path.relative_to(root)
|
||||
parts = rel.parts
|
||||
category = parts[0] if parts else ""
|
||||
skill_name = parts[1] if len(parts) > 1 else ""
|
||||
return category, skill_name, rel.as_posix()
|
||||
|
||||
|
||||
def _read_text(path: Path) -> str | None:
|
||||
"""Read UTF-8 text preserving original line endings; skip binary/bad files."""
|
||||
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8", newline="") as fh:
|
||||
return fh.read()
|
||||
except (UnicodeDecodeError, OSError):
|
||||
return None
|
||||
|
||||
|
||||
def _write_text(path: Path, text: str) -> None:
|
||||
with open(path, "w", encoding="utf-8", newline="") as fh:
|
||||
fh.write(text)
|
||||
|
||||
|
||||
# ── synchronous workers (run via asyncio.to_thread; never block the loop) ─────
|
||||
|
||||
|
||||
def _iter_skill_files(root: Path, skill: str | None = None) -> list[Path]:
|
||||
"""All ``*.md`` / ``*.py`` under ``{root}/{public,custom}``.
|
||||
|
||||
Hidden directories (``.address-edits`` backups, ``.history``) are skipped.
|
||||
When *skill* is given, only that skill's directory is walked.
|
||||
"""
|
||||
|
||||
files: list[Path] = []
|
||||
for sub in _SCAN_SUBDIRS:
|
||||
base = root / sub
|
||||
if skill:
|
||||
base = base / skill
|
||||
if not base.is_dir():
|
||||
continue
|
||||
for p in base.rglob("*"):
|
||||
if not p.is_file() or p.suffix.lower() not in _SCAN_EXTS:
|
||||
continue
|
||||
try:
|
||||
rel_parts = p.relative_to(root).parts
|
||||
except ValueError:
|
||||
continue
|
||||
if any(part.startswith(".") for part in rel_parts):
|
||||
continue
|
||||
files.append(p)
|
||||
return files
|
||||
|
||||
|
||||
def _group_files_by_skill(root: Path, files: list[Path]) -> dict[tuple[str, str], list[Path]]:
|
||||
"""Bucket scanned files under their owning ``(category, skill_name)``."""
|
||||
|
||||
buckets: dict[tuple[str, str], list[Path]] = {}
|
||||
for p in files:
|
||||
category, skill_name, _ = _skill_of(root, p)
|
||||
buckets.setdefault((category, skill_name), []).append(p)
|
||||
return buckets
|
||||
|
||||
|
||||
def _occurrence_kind(value: str) -> str:
|
||||
return infer_kind(value) or "url"
|
||||
|
||||
|
||||
def _merge_occurrences(groups: dict[str, AddressGroup], pairs: list[tuple[str, OccurrenceModel]]) -> None:
|
||||
"""Merge ``(address, occurrence)`` pairs into the aggregate group map."""
|
||||
|
||||
for address, occ in pairs:
|
||||
g = groups.get(address)
|
||||
if g is None:
|
||||
g = AddressGroup(address=address, kind=_occurrence_kind(address), count=0, occurrences=[])
|
||||
groups[address] = g
|
||||
g.count += 1
|
||||
g.occurrences.append(occ)
|
||||
|
||||
|
||||
def _plan_apply(root: Path, mapping: dict[str, str]) -> tuple[list[tuple[Path, str, int, str]], int]:
|
||||
"""Compute the rewrite plan without touching disk → (plan, current_total)."""
|
||||
|
||||
plan: list[tuple[Path, str, int, str]] = []
|
||||
current_total = 0
|
||||
for path in _iter_skill_files(root):
|
||||
text = _read_text(path)
|
||||
if text is None:
|
||||
continue
|
||||
new_text, cnt = apply_replacements(text, mapping, VALID_KINDS)
|
||||
if cnt <= 0:
|
||||
continue
|
||||
if not _is_within(root, path):
|
||||
raise HTTPException(status_code=400, detail=f"拒绝写入技能目录之外的文件:{path}")
|
||||
_, _, rel = _skill_of(root, path)
|
||||
plan.append((path, new_text, cnt, rel))
|
||||
current_total += cnt
|
||||
return plan, current_total
|
||||
|
||||
|
||||
def _backup_file(root: Path, backup_id: str, path: Path, rel: str) -> None:
|
||||
dest = root / _BACKUP_DIRNAME / backup_id / rel
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(path, dest)
|
||||
|
||||
|
||||
_MANIFEST_SUFFIX = ".manifest.json"
|
||||
|
||||
|
||||
def _manifest_path(root: Path, backup_id: str) -> Path:
|
||||
"""Manifest lives as a *sibling* of the backup dir (``{id}.manifest.json``),
|
||||
not inside it — so it is neither restored back into the skills tree on
|
||||
rollback nor counted as a backed-up file."""
|
||||
|
||||
return root / _BACKUP_DIRNAME / f"{backup_id}{_MANIFEST_SUFFIX}"
|
||||
|
||||
|
||||
def _write_manifest(
|
||||
root: Path,
|
||||
backup_id: str,
|
||||
norm: list[tuple[str, str, str]],
|
||||
changed: list["ChangedFile"],
|
||||
replaced_total: int,
|
||||
created_at: str,
|
||||
) -> None:
|
||||
"""Record what this edit changed so the UI can show a meaningful, per-edit
|
||||
history (原地址 → 新地址, 文件数, 处数) and roll a single record back."""
|
||||
|
||||
data = {
|
||||
"created_at": created_at,
|
||||
"replaced_total": replaced_total,
|
||||
"replacements": [{"from": frm, "to": to, "kind": kind} for frm, to, kind in norm],
|
||||
"changed_files": [{"file": c.file, "count": c.count} for c in changed],
|
||||
}
|
||||
try: # best-effort: a metadata write failure must not fail the apply itself
|
||||
path = _manifest_path(root, backup_id)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(path, "w", encoding="utf-8") as fh:
|
||||
json.dump(data, fh, ensure_ascii=False)
|
||||
except OSError:
|
||||
logger.warning("Failed to write skill-address backup manifest for %s", backup_id, exc_info=True)
|
||||
|
||||
|
||||
def _read_manifest(root: Path, backup_id: str) -> dict | None:
|
||||
try:
|
||||
with open(_manifest_path(root, backup_id), "r", encoding="utf-8") as fh:
|
||||
return json.load(fh)
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return None
|
||||
|
||||
|
||||
# ── scan extraction shared by one-shot + stream ───────────────────────────────
|
||||
|
||||
|
||||
def _scan_files_with_addresses(root: Path, files: list[Path], wanted: tuple[str, ...]) -> tuple[list[tuple[str, OccurrenceModel]], int]:
|
||||
"""Scan files → ([(address, occurrence)], files_read)."""
|
||||
|
||||
pairs: list[tuple[str, OccurrenceModel]] = []
|
||||
read = 0
|
||||
for path in files:
|
||||
text = _read_text(path)
|
||||
if text is None:
|
||||
continue
|
||||
read += 1
|
||||
category, skill_name, rel = _skill_of(root, path)
|
||||
for o in scan_occurrences(text, wanted):
|
||||
pairs.append(
|
||||
(
|
||||
o.value,
|
||||
OccurrenceModel(skill=skill_name, category=category, file=rel, line=o.line, col=o.col, snippet=o.snippet),
|
||||
)
|
||||
)
|
||||
return pairs, read
|
||||
|
||||
|
||||
def _ordered_groups(groups: dict[str, AddressGroup]) -> list[AddressGroup]:
|
||||
return sorted(groups.values(), key=lambda g: (-g.count, g.address))
|
||||
|
||||
|
||||
def _build_mapping(replacements: list[Replacement]) -> tuple[dict[str, str], list[tuple[str, str, str]], list[str]]:
|
||||
"""Validate replacements → ``(mapping, [(from,to,kind)], warnings)``.
|
||||
|
||||
Drops (with a warning) any entry that is empty, duplicated, not a recognized
|
||||
URL/IP, a no-op (``from == to``), or whose ``to`` is malformed for its kind.
|
||||
"""
|
||||
|
||||
mapping: dict[str, str] = {}
|
||||
norm: list[tuple[str, str, str]] = []
|
||||
warnings: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for r in replacements:
|
||||
frm = r.from_.strip()
|
||||
to = r.to.strip()
|
||||
if not frm:
|
||||
continue
|
||||
if frm in seen:
|
||||
warnings.append(f"重复的原始地址已忽略:{frm}")
|
||||
continue
|
||||
kind = (r.kind or "").strip().lower() or infer_kind(frm)
|
||||
if kind not in VALID_KINDS:
|
||||
warnings.append(f"无法识别为 URL/IP,已忽略:{frm}")
|
||||
continue
|
||||
if to == frm:
|
||||
warnings.append(f"替换值与原值相同,已忽略:{frm}")
|
||||
continue
|
||||
if not is_valid_replacement(kind, to):
|
||||
warnings.append(f"替换值不是合法的 {kind},已忽略:{to}")
|
||||
continue
|
||||
seen.add(frm)
|
||||
mapping[frm] = to
|
||||
norm.append((frm, to, kind))
|
||||
return mapping, norm, warnings
|
||||
|
||||
|
||||
def _ndjson(obj: dict) -> bytes:
|
||||
return (json.dumps(obj, ensure_ascii=False) + "\n").encode("utf-8")
|
||||
|
||||
|
||||
# ── routes: scan ─────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@router.get("/scan", response_model=ScanResponse)
|
||||
async def scan_addresses(
|
||||
request: Request,
|
||||
kinds: str | None = Query(default=None, description="逗号分隔:url,ip(默认全部)"),
|
||||
skill: str | None = Query(default=None, description="仅扫描指定技能目录"),
|
||||
) -> ScanResponse:
|
||||
await _require_admin(request)
|
||||
root = _skills_root()
|
||||
wanted = normalize_kinds(kinds)
|
||||
files = await asyncio.to_thread(_iter_skill_files, root, skill)
|
||||
buckets = _group_files_by_skill(root, files)
|
||||
groups: dict[str, AddressGroup] = {}
|
||||
scanned = 0
|
||||
for paths in buckets.values():
|
||||
pairs, read = await asyncio.to_thread(_scan_files_with_addresses, root, paths, wanted)
|
||||
_merge_occurrences(groups, pairs)
|
||||
scanned += read
|
||||
return ScanResponse(
|
||||
groups=_ordered_groups(groups),
|
||||
scanned_files=scanned,
|
||||
scanned_skills=len(buckets),
|
||||
kinds=list(wanted),
|
||||
scanned_at=datetime.now(UTC).isoformat(),
|
||||
)
|
||||
|
||||
|
||||
@router.get("/scan/stream")
|
||||
async def scan_addresses_stream(
|
||||
request: Request,
|
||||
kinds: str | None = Query(default=None),
|
||||
skill: str | None = Query(default=None),
|
||||
) -> StreamingResponse:
|
||||
"""Scan with per-skill NDJSON progress.
|
||||
|
||||
Emits ``{"type":"start","total":N}`` then one
|
||||
``{"type":"progress","done":i,"total":N,"skill":name}`` per skill, finally
|
||||
``{"type":"result", ...ScanResponse...}``.
|
||||
"""
|
||||
|
||||
await _require_admin(request)
|
||||
root = _skills_root()
|
||||
wanted = normalize_kinds(kinds)
|
||||
|
||||
async def gen() -> AsyncIterator[bytes]:
|
||||
files = await asyncio.to_thread(_iter_skill_files, root, skill)
|
||||
buckets = _group_files_by_skill(root, files)
|
||||
total = len(buckets)
|
||||
yield _ndjson({"type": "start", "total": total})
|
||||
groups: dict[str, AddressGroup] = {}
|
||||
scanned = 0
|
||||
done = 0
|
||||
for (category, skill_name), paths in buckets.items():
|
||||
pairs, read = await asyncio.to_thread(_scan_files_with_addresses, root, paths, wanted)
|
||||
_merge_occurrences(groups, pairs)
|
||||
scanned += read
|
||||
done += 1
|
||||
yield _ndjson({"type": "progress", "done": done, "total": total, "skill": skill_name or category})
|
||||
result = ScanResponse(
|
||||
groups=_ordered_groups(groups),
|
||||
scanned_files=scanned,
|
||||
scanned_skills=total,
|
||||
kinds=list(wanted),
|
||||
scanned_at=datetime.now(UTC).isoformat(),
|
||||
)
|
||||
yield _ndjson({"type": "result", **result.model_dump()})
|
||||
|
||||
return StreamingResponse(gen(), media_type="application/x-ndjson")
|
||||
|
||||
|
||||
# ── routes: preview ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _compute_preview(root: Path, mapping: dict[str, str], norm: list[tuple[str, str, str]]) -> tuple[list[PreviewItem], int]:
|
||||
counts: dict[str, int] = {frm: 0 for frm, _, _ in norm}
|
||||
files: dict[str, list[PreviewFileHit]] = {frm: [] for frm, _, _ in norm}
|
||||
total = 0
|
||||
if mapping:
|
||||
for path in _iter_skill_files(root):
|
||||
text = _read_text(path)
|
||||
if text is None:
|
||||
continue
|
||||
_, _, rel = _skill_of(root, path)
|
||||
for o in scan_occurrences(text, VALID_KINDS):
|
||||
if o.value not in mapping:
|
||||
continue
|
||||
to = mapping[o.value]
|
||||
counts[o.value] += 1
|
||||
total += 1
|
||||
files[o.value].append(PreviewFileHit(file=rel, line=o.line, before=o.snippet, after=o.snippet.replace(o.value, to)))
|
||||
items = [PreviewItem(from_=frm, to=to, kind=kind, count=counts[frm], files=files[frm]) for frm, to, kind in norm]
|
||||
return items, total
|
||||
|
||||
|
||||
@router.post("/preview", response_model=PreviewResponse)
|
||||
async def preview_replacements(request: Request, body: PreviewRequest) -> PreviewResponse:
|
||||
await _require_admin(request)
|
||||
root = _skills_root()
|
||||
mapping, norm, warnings = _build_mapping(body.replacements)
|
||||
items, total = await asyncio.to_thread(_compute_preview, root, mapping, norm)
|
||||
return PreviewResponse(items=items, total=total, warnings=warnings, expected_total=total)
|
||||
|
||||
|
||||
# ── routes: apply ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _new_backup_id() -> str:
|
||||
return datetime.now(UTC).strftime("%Y%m%d-%H%M%S-%f")
|
||||
|
||||
|
||||
@router.post("/apply", response_model=ApplyResponse)
|
||||
async def apply_replacements_route(request: Request, body: ApplyRequest) -> ApplyResponse:
|
||||
await _require_admin(request)
|
||||
root = _skills_root()
|
||||
mapping, norm, warnings = _build_mapping(body.replacements)
|
||||
if not mapping:
|
||||
return ApplyResponse(changed_files=[], replaced_total=0, backup_id=None, warnings=warnings)
|
||||
|
||||
plan, current_total = await asyncio.to_thread(_plan_apply, root, mapping)
|
||||
if body.expected_total is not None and body.expected_total != current_total:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail=f"文件自上次扫描后已变化(预期 {body.expected_total} 处,现为 {current_total} 处),请重新扫描后再应用。",
|
||||
)
|
||||
if not plan:
|
||||
return ApplyResponse(changed_files=[], replaced_total=0, backup_id=None, warnings=warnings)
|
||||
|
||||
backup_id: str | None = None
|
||||
if body.backup:
|
||||
backup_id = _new_backup_id()
|
||||
for path, _new, _cnt, rel in plan:
|
||||
await asyncio.to_thread(_backup_file, root, backup_id, path, rel)
|
||||
|
||||
changed: list[ChangedFile] = []
|
||||
replaced_total = 0
|
||||
for path, new_text, cnt, rel in plan:
|
||||
await asyncio.to_thread(_write_text, path, new_text)
|
||||
changed.append(ChangedFile(file=rel, count=cnt))
|
||||
replaced_total += cnt
|
||||
|
||||
if backup_id:
|
||||
await asyncio.to_thread(_write_manifest, root, backup_id, norm, changed, replaced_total, datetime.now(UTC).isoformat())
|
||||
|
||||
await _refresh_cache_safe()
|
||||
logger.info("Skill address apply: %d files, %d replacements, backup=%s", len(changed), replaced_total, backup_id)
|
||||
return ApplyResponse(changed_files=changed, replaced_total=replaced_total, backup_id=backup_id, warnings=warnings)
|
||||
|
||||
|
||||
@router.post("/apply/stream")
|
||||
async def apply_replacements_stream(request: Request, body: ApplyRequest) -> StreamingResponse:
|
||||
"""Apply with per-file NDJSON progress.
|
||||
|
||||
Validation + the optimistic-lock check happen *before* streaming (so a 409
|
||||
is a normal HTTP error). Then: ``{"type":"start","total_files":M,...}`` →
|
||||
one ``{"type":"progress","done":k,...}`` per written file →
|
||||
``{"type":"result", ...ApplyResponse...}``.
|
||||
"""
|
||||
|
||||
await _require_admin(request)
|
||||
root = _skills_root()
|
||||
mapping, norm, warnings = _build_mapping(body.replacements)
|
||||
|
||||
plan: list[tuple[Path, str, int, str]] = []
|
||||
current_total = 0
|
||||
if mapping:
|
||||
plan, current_total = await asyncio.to_thread(_plan_apply, root, mapping)
|
||||
if body.expected_total is not None and body.expected_total != current_total:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail=f"文件自上次扫描后已变化(预期 {body.expected_total} 处,现为 {current_total} 处),请重新扫描后再应用。",
|
||||
)
|
||||
|
||||
async def gen() -> AsyncIterator[bytes]:
|
||||
if not plan:
|
||||
result = ApplyResponse(changed_files=[], replaced_total=0, backup_id=None, warnings=warnings)
|
||||
yield _ndjson({"type": "result", **result.model_dump()})
|
||||
return
|
||||
|
||||
backup_id: str | None = None
|
||||
if body.backup:
|
||||
backup_id = _new_backup_id()
|
||||
|
||||
yield _ndjson({"type": "start", "total_files": len(plan), "total_replacements": current_total})
|
||||
|
||||
changed: list[ChangedFile] = []
|
||||
replaced_total = 0
|
||||
done = 0
|
||||
for path, new_text, cnt, rel in plan:
|
||||
if backup_id:
|
||||
await asyncio.to_thread(_backup_file, root, backup_id, path, rel)
|
||||
await asyncio.to_thread(_write_text, path, new_text)
|
||||
changed.append(ChangedFile(file=rel, count=cnt))
|
||||
replaced_total += cnt
|
||||
done += 1
|
||||
yield _ndjson({"type": "progress", "done": done, "total_files": len(plan), "file": rel, "count": cnt})
|
||||
|
||||
if backup_id:
|
||||
await asyncio.to_thread(_write_manifest, root, backup_id, norm, changed, replaced_total, datetime.now(UTC).isoformat())
|
||||
|
||||
await _refresh_cache_safe()
|
||||
logger.info("Skill address apply(stream): %d files, %d replacements, backup=%s", len(changed), replaced_total, backup_id)
|
||||
result = ApplyResponse(changed_files=changed, replaced_total=replaced_total, backup_id=backup_id, warnings=warnings)
|
||||
yield _ndjson({"type": "result", **result.model_dump()})
|
||||
|
||||
return StreamingResponse(gen(), media_type="application/x-ndjson")
|
||||
|
||||
|
||||
# ── routes: backups / rollback ───────────────────────────────────────────────
|
||||
|
||||
|
||||
def _list_backups(root: Path) -> list[BackupEntry]:
|
||||
backup_root = root / _BACKUP_DIRNAME
|
||||
entries: list[BackupEntry] = []
|
||||
if backup_root.is_dir():
|
||||
for d in backup_root.iterdir():
|
||||
if not d.is_dir():
|
||||
continue
|
||||
file_count = sum(1 for p in d.rglob("*") if p.is_file())
|
||||
manifest = _read_manifest(root, d.name)
|
||||
created = ""
|
||||
replaced_total = 0
|
||||
replacements: list[ReplacementSummary] = []
|
||||
if manifest:
|
||||
created = str(manifest.get("created_at") or "")
|
||||
replaced_total = int(manifest.get("replaced_total") or 0)
|
||||
replacements = [
|
||||
ReplacementSummary(from_=str(r.get("from", "")), to=str(r.get("to", "")), kind=str(r.get("kind", "")))
|
||||
for r in manifest.get("replacements", [])
|
||||
if isinstance(r, dict)
|
||||
]
|
||||
if not created:
|
||||
try:
|
||||
created = datetime.fromtimestamp(d.stat().st_mtime, UTC).isoformat()
|
||||
except OSError:
|
||||
created = ""
|
||||
entries.append(
|
||||
BackupEntry(
|
||||
backup_id=d.name,
|
||||
files=file_count,
|
||||
created_at=created,
|
||||
replaced_total=replaced_total,
|
||||
replacements=replacements,
|
||||
)
|
||||
)
|
||||
entries.sort(key=lambda e: e.backup_id, reverse=True)
|
||||
return entries
|
||||
|
||||
|
||||
@router.get("/backups", response_model=BackupListResponse)
|
||||
async def list_backups(request: Request) -> BackupListResponse:
|
||||
await _require_admin(request)
|
||||
root = _skills_root()
|
||||
entries = await asyncio.to_thread(_list_backups, root)
|
||||
return BackupListResponse(backups=entries)
|
||||
|
||||
|
||||
def _restore_backup(root: Path, backup_root: Path) -> list[str]:
|
||||
restored: list[str] = []
|
||||
for src in backup_root.rglob("*"):
|
||||
if not src.is_file():
|
||||
continue
|
||||
rel = src.relative_to(backup_root)
|
||||
dest = root / rel
|
||||
if not _is_within(root, dest):
|
||||
continue
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(src, dest)
|
||||
restored.append(rel.as_posix())
|
||||
return restored
|
||||
|
||||
|
||||
@router.post("/rollback/{backup_id}", response_model=RollbackResponse)
|
||||
async def rollback(request: Request, backup_id: str) -> RollbackResponse:
|
||||
await _require_admin(request)
|
||||
root = _skills_root()
|
||||
if "/" in backup_id or "\\" in backup_id or backup_id in (".", ".."):
|
||||
raise HTTPException(status_code=400, detail="非法的备份编号")
|
||||
backup_root = (root / _BACKUP_DIRNAME / backup_id).resolve()
|
||||
if not _is_within(root, backup_root) or not backup_root.is_dir():
|
||||
raise HTTPException(status_code=404, detail="备份不存在")
|
||||
|
||||
restored = await asyncio.to_thread(_restore_backup, root, backup_root)
|
||||
await _refresh_cache_safe()
|
||||
logger.info("Skill address rollback: backup=%s, restored %d files", backup_id, len(restored))
|
||||
return RollbackResponse(backup_id=backup_id, restored_files=restored)
|
||||
|
||||
|
||||
# ── misc ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def _refresh_cache_safe() -> None:
|
||||
try:
|
||||
await refresh_skills_system_prompt_cache_async()
|
||||
except Exception: # noqa: BLE001 — cache refresh is best-effort
|
||||
logger.warning("Failed to refresh skills prompt cache", exc_info=True)
|
||||
@@ -222,6 +222,10 @@ _CONTEXT_CONFIGURABLE_KEYS: frozenset[str] = frozenset(
|
||||
"model_name",
|
||||
"mode",
|
||||
"thinking_enabled",
|
||||
# 强制关闭思考(内网 vLLM/Qwen 模型只声明 supports_thinking 时,单靠
|
||||
# thinking_enabled=False 不会注入 enable_thinking=false)。透传到 run config
|
||||
# 后由 lead agent 交给 create_chat_model(force_disable_thinking=...)。
|
||||
"thinking_force_disabled",
|
||||
"reasoning_effort",
|
||||
"rag_mode_enabled",
|
||||
"writing_mode",
|
||||
|
||||
Reference in new issue
Block a user