feat: 新增技能地址管理与开放问答接口,完善任务工作区
- 新增「技能地址管理」页面与 /api/skill-addresses 路由(扫描/预览/批量替换技能内地址) - 完善 /api/open/chat 开放问答接口与模型工厂、网关配置 - 调整任务工作区、侧边栏与路由 - 将本地 0622/ 临时目录与 __pycache__/ 加入忽略,不提交 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
32476c7ad4
commit
6b480fdbe9
15 files changed
+1749
-40
No files matched your search
@@ -69,6 +69,7 @@ from app.gateway.routers import (
|
||||
runs,
|
||||
scheduled_tasks,
|
||||
sensitive_words,
|
||||
skill_addresses,
|
||||
skills,
|
||||
suggestions,
|
||||
system_settings,
|
||||
@@ -1359,6 +1360,9 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
# Sensitive-word map (敏感词管理 / 脱敏词) at /api/sensitive-words
|
||||
app.include_router(sensitive_words.router)
|
||||
|
||||
# Skill URL/IP batch replace (技能地址管理) at /api/skill-addresses
|
||||
app.include_router(skill_addresses.router)
|
||||
|
||||
# 大屏绘制智能体独立页面会话(按 taskId 存取,不分权)at /api/dashboard-sessions
|
||||
app.include_router(dashboard_sessions.router)
|
||||
|
||||
|
||||
@@ -33,12 +33,16 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import uuid
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any, Literal
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from fastapi.responses import StreamingResponse
|
||||
from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, SystemMessage
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
@@ -167,15 +171,15 @@ def _extract_answer(result: Any, *, strip_think: bool) -> str:
|
||||
return answer
|
||||
|
||||
|
||||
@router.post("/chat", response_model=OpenChatResponse)
|
||||
async def open_chat(body: OpenChatRequest) -> OpenChatResponse:
|
||||
"""指定模型 + 智能体做一次问答(无状态、可关闭思考)。"""
|
||||
lc_messages = _build_lc_messages(body)
|
||||
# 流式响应每隔这么久没有真实事件就发一个 SSE 注释心跳,持续制造下行字节,
|
||||
# 让前置反向代理(nginx proxy_read_timeout 默认 60s 等)不会把长任务判定成空闲超时。
|
||||
_HEARTBEAT_SECONDS = 15.0
|
||||
|
||||
|
||||
def _build_run_context(body: OpenChatRequest) -> tuple[dict[str, Any], str | None, str]:
|
||||
"""把请求体翻译成 (config, agent_id, thread_id)。两个端点共用,确保关思考等行为一致。"""
|
||||
agent_id = _resolve_agent_id(body.agent_id, body.agent_name)
|
||||
|
||||
thread_id = body.thread_id or f"open-{uuid.uuid4()}"
|
||||
|
||||
configurable: dict[str, Any] = {
|
||||
"thread_id": thread_id,
|
||||
"model_name": body.model_name,
|
||||
@@ -187,8 +191,26 @@ async def open_chat(body: OpenChatRequest) -> OpenChatResponse:
|
||||
}
|
||||
if agent_id:
|
||||
configurable["agent_id"] = agent_id
|
||||
|
||||
config: dict[str, Any] = {"configurable": configurable, "recursion_limit": body.recursion_limit}
|
||||
return config, agent_id, thread_id
|
||||
|
||||
|
||||
def _sse(event: str | None, data: Any) -> str:
|
||||
"""序列化一条 SSE 消息。event=None 时只发 data 行。"""
|
||||
payload = json.dumps(data, ensure_ascii=False)
|
||||
head = f"event: {event}\n" if event else ""
|
||||
return f"{head}data: {payload}\n\n"
|
||||
|
||||
|
||||
@router.post("/chat", response_model=OpenChatResponse)
|
||||
async def open_chat(body: OpenChatRequest) -> OpenChatResponse:
|
||||
"""指定模型 + 智能体做一次问答(无状态、可关闭思考)。
|
||||
|
||||
同步阻塞版:整段调研跑完才返回。**长任务(联网调研)请改用 /chat/stream**——
|
||||
本端点期间不产生任何下行字节,容易被前置反代的 proxy_read_timeout 掐成 504。
|
||||
"""
|
||||
lc_messages = _build_lc_messages(body)
|
||||
config, agent_id, thread_id = _build_run_context(body)
|
||||
|
||||
try:
|
||||
agent = make_lead_agent(config)
|
||||
@@ -211,6 +233,103 @@ async def open_chat(body: OpenChatRequest) -> OpenChatResponse:
|
||||
)
|
||||
|
||||
|
||||
async def _open_chat_event_stream(body: OpenChatRequest) -> AsyncIterator[str]:
|
||||
"""驱动智能体并产出 SSE 事件流。
|
||||
|
||||
- 边跑边转发 AI 文本增量(``event: delta``,仅供实时观看,客户端可忽略);
|
||||
- 没有真实事件时每 ``_HEARTBEAT_SECONDS`` 发一个心跳注释行保活;
|
||||
- 全部跑完后用 ``event: result`` 给出最终(已按需剥离 <think> 的)答案;
|
||||
- 出错则 ``event: error``。
|
||||
|
||||
用「后台生产任务 + 队列 + 超时取队列」的方式把「智能体进度」与「定时心跳」解耦,
|
||||
所以即便某次工具调用静默很久,心跳仍照常下行,连接不会被反代判空闲。
|
||||
"""
|
||||
lc_messages = _build_lc_messages(body)
|
||||
config, agent_id, thread_id = _build_run_context(body)
|
||||
strip_think = body.strip_think and not body.thinking_enabled
|
||||
|
||||
queue: asyncio.Queue[tuple[str, Any]] = asyncio.Queue()
|
||||
final_state: dict[str, Any] = {}
|
||||
|
||||
async def produce() -> None:
|
||||
nonlocal final_state
|
||||
try:
|
||||
agent = make_lead_agent(config)
|
||||
async for mode, chunk in agent.astream({"messages": lc_messages}, config=config, stream_mode=["values", "messages"]):
|
||||
if mode == "values" and isinstance(chunk, dict):
|
||||
final_state = chunk
|
||||
await queue.put(("progress", {"messages": len(chunk.get("messages") or [])}))
|
||||
elif mode == "messages":
|
||||
msg = chunk[0] if isinstance(chunk, tuple) and chunk else chunk
|
||||
# 仅转发 AI 正文增量;DeepSeek 等把思考放在 additional_kwargs.reasoning_content,
|
||||
# 不在 content 里,所以这里天然不会泄漏 <think>。
|
||||
if isinstance(msg, AIMessage):
|
||||
text = _extract_text(getattr(msg, "content", "")).strip()
|
||||
if text:
|
||||
await queue.put(("delta", {"text": text}))
|
||||
except Exception as exc: # noqa: BLE001 — 开放接口需把内部错误归一成 error 事件
|
||||
logger.exception("open_chat_stream 调用智能体失败 (model=%s, agent=%s)", body.model_name, agent_id)
|
||||
await queue.put(("error", {"detail": f"智能体调用失败:{exc}"}))
|
||||
finally:
|
||||
await queue.put(("__end__", None))
|
||||
|
||||
task = asyncio.create_task(produce())
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
event, data = await asyncio.wait_for(queue.get(), timeout=_HEARTBEAT_SECONDS)
|
||||
except asyncio.TimeoutError:
|
||||
yield ": keepalive\n\n" # SSE 注释行,客户端忽略,仅用于保活
|
||||
continue
|
||||
if event == "__end__":
|
||||
break
|
||||
if event == "error":
|
||||
yield _sse("error", data)
|
||||
return
|
||||
yield _sse(event, data)
|
||||
|
||||
answer = _extract_answer(final_state, strip_think=strip_think)
|
||||
if not answer:
|
||||
yield _sse("error", {"detail": "智能体未返回任何文本答案。"})
|
||||
return
|
||||
yield _sse(
|
||||
"result",
|
||||
{
|
||||
"answer": answer,
|
||||
"model_name": body.model_name,
|
||||
"agent_name": body.agent_name,
|
||||
"agent_id": agent_id,
|
||||
"thinking_enabled": body.thinking_enabled,
|
||||
"thread_id": thread_id,
|
||||
},
|
||||
)
|
||||
finally:
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
|
||||
|
||||
@router.post("/chat/stream")
|
||||
async def open_chat_stream(body: OpenChatRequest) -> StreamingResponse:
|
||||
"""流式版 /chat:边跑边发心跳保活,跑完用 ``event: result`` 给最终答案。
|
||||
|
||||
解决两件事:(1) 长任务(联网调研、报告)同步阻塞会被前置反代 proxy_read_timeout
|
||||
掐成 504;流式持续有字节下行,代理不再判定空闲超时。(2) 客户端只需读到
|
||||
``event: result`` 解析 ``answer`` 即可,中间的 ``delta``/``progress``/心跳都可忽略。
|
||||
"""
|
||||
# 入参校验放在流外:非法请求直接抛 4xx,而不是开了流再报错。
|
||||
_build_lc_messages(body)
|
||||
return StreamingResponse(
|
||||
_open_chat_event_stream(body),
|
||||
media_type="text/event-stream",
|
||||
headers={
|
||||
"Cache-Control": "no-cache",
|
||||
"Connection": "keep-alive",
|
||||
# 关掉 nginx 对该响应的缓冲,确保 SSE/心跳实时下行(即便前置 nginx 默认开缓冲)。
|
||||
"X-Accel-Buffering": "no",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/agents")
|
||||
async def open_list_agents() -> dict[str, Any]:
|
||||
"""列出可用智能体的名称(方便知道 agent_name 该填什么)。"""
|
||||
|
||||
@@ -0,0 +1,631 @@
|
||||
"""技能地址(URL/IP)批量替换 API (技能地址管理).
|
||||
|
||||
Scans the **running** skills directory (``skills/{public,custom}``) — the files
|
||||
the agent actually reads — for ``http(s)://`` URLs and IPv4 (``ip[:port]``)
|
||||
addresses, aggregates each unique address with every place it appears (across
|
||||
``.md`` and ``.py``, across files), and lets an admin rewrite them in bulk.
|
||||
|
||||
Why this shape: the same address routinely lives in both a skill's ``SKILL.md``
|
||||
(documented endpoint) and its ``scripts/*.py`` (the actual call), so the natural
|
||||
unit is *"one address → replace all of its occurrences at once"*. The pure
|
||||
scan/replace logic lives in ``deerflow.skills.address_scan`` (harness layer,
|
||||
substring-safe span replacement); this router only walks the tree, enforces
|
||||
admin auth + path safety, backs up before writing, and refreshes the skills
|
||||
system-prompt cache so edits take effect immediately.
|
||||
|
||||
**Scale / availability (150+ skills):** every filesystem walk / read / write is
|
||||
offloaded to a worker thread via ``asyncio.to_thread`` so the event loop is
|
||||
never blocked while scanning or rewriting — the Gateway stays responsive. The
|
||||
streaming endpoints additionally emit NDJSON progress (``已处理 X / 共 N``) so the
|
||||
UI can render a progress bar for both the scan and the apply phases.
|
||||
|
||||
Routes (prefix ``/api/skill-addresses``, **admin only**):
|
||||
GET /scan?kinds=url,ip&skill=<name> read-only aggregated scan (one shot)
|
||||
GET /scan/stream?kinds=...&skill=... scan with per-skill NDJSON progress
|
||||
POST /preview dry-run diff of a replacement set
|
||||
POST /apply write changes (one shot, with backup)
|
||||
POST /apply/stream write changes with per-file progress
|
||||
GET /backups list restorable backups (newest first)
|
||||
POST /rollback/{backup_id} restore a backup
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import shutil
|
||||
from collections.abc import AsyncIterator
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query, Request
|
||||
from fastapi.responses import StreamingResponse
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from app.gateway.deps import get_optional_user_from_request
|
||||
from deerflow.agents.lead_agent.prompt import refresh_skills_system_prompt_cache_async
|
||||
from deerflow.skills.address_scan import (
|
||||
VALID_KINDS,
|
||||
apply_replacements,
|
||||
infer_kind,
|
||||
is_valid_replacement,
|
||||
normalize_kinds,
|
||||
scan_occurrences,
|
||||
)
|
||||
from deerflow.skills.storage import get_or_new_skill_storage
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/skill-addresses", tags=["skill-addresses"])
|
||||
|
||||
_BACKUP_DIRNAME = ".address-edits"
|
||||
_SCAN_SUBDIRS = ("public", "custom")
|
||||
_SCAN_EXTS = (".md", ".py")
|
||||
|
||||
|
||||
# ── schemas ────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class OccurrenceModel(BaseModel):
|
||||
skill: str
|
||||
category: str # "public" | "custom"
|
||||
file: str # POSIX path relative to the skills root
|
||||
line: int
|
||||
col: int
|
||||
snippet: str
|
||||
|
||||
|
||||
class AddressGroup(BaseModel):
|
||||
address: str
|
||||
kind: str # "url" | "ip"
|
||||
count: int
|
||||
occurrences: list[OccurrenceModel]
|
||||
|
||||
|
||||
class ScanResponse(BaseModel):
|
||||
groups: list[AddressGroup]
|
||||
scanned_files: int
|
||||
scanned_skills: int
|
||||
kinds: list[str]
|
||||
scanned_at: str
|
||||
|
||||
|
||||
class Replacement(BaseModel):
|
||||
model_config = ConfigDict(populate_by_name=True)
|
||||
|
||||
from_: str = Field(alias="from", min_length=1)
|
||||
to: str = Field(min_length=1)
|
||||
kind: str | None = None
|
||||
|
||||
|
||||
class PreviewRequest(BaseModel):
|
||||
replacements: list[Replacement] = Field(default_factory=list)
|
||||
|
||||
|
||||
class PreviewFileHit(BaseModel):
|
||||
file: str
|
||||
line: int
|
||||
before: str
|
||||
after: str
|
||||
|
||||
|
||||
class PreviewItem(BaseModel):
|
||||
model_config = ConfigDict(populate_by_name=True)
|
||||
|
||||
from_: str = Field(serialization_alias="from")
|
||||
to: str
|
||||
kind: str
|
||||
count: int
|
||||
files: list[PreviewFileHit]
|
||||
|
||||
|
||||
class PreviewResponse(BaseModel):
|
||||
items: list[PreviewItem]
|
||||
total: int
|
||||
warnings: list[str]
|
||||
expected_total: int
|
||||
|
||||
|
||||
class ApplyRequest(BaseModel):
|
||||
replacements: list[Replacement] = Field(default_factory=list)
|
||||
expected_total: int | None = None
|
||||
backup: bool = True
|
||||
|
||||
|
||||
class ChangedFile(BaseModel):
|
||||
file: str
|
||||
count: int
|
||||
|
||||
|
||||
class ApplyResponse(BaseModel):
|
||||
changed_files: list[ChangedFile]
|
||||
replaced_total: int
|
||||
backup_id: str | None
|
||||
warnings: list[str]
|
||||
|
||||
|
||||
class BackupEntry(BaseModel):
|
||||
backup_id: str
|
||||
files: int
|
||||
created_at: str
|
||||
|
||||
|
||||
class BackupListResponse(BaseModel):
|
||||
backups: list[BackupEntry]
|
||||
|
||||
|
||||
class RollbackResponse(BaseModel):
|
||||
backup_id: str
|
||||
restored_files: list[str]
|
||||
|
||||
|
||||
# ── auth / path helpers ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def _require_admin(request: Request) -> None:
|
||||
"""Admin-only. Lets the call through when auth is disabled (no user)."""
|
||||
|
||||
user = await get_optional_user_from_request(request)
|
||||
if user is None:
|
||||
return
|
||||
if getattr(user, "system_role", None) != "admin":
|
||||
raise HTTPException(status_code=403, detail="技能地址管理仅限管理员")
|
||||
|
||||
|
||||
def _skills_root() -> Path:
|
||||
return Path(get_or_new_skill_storage().get_skills_root_path()).resolve()
|
||||
|
||||
|
||||
def _is_within(root: Path, p: Path) -> bool:
|
||||
try:
|
||||
p.resolve().relative_to(root)
|
||||
return True
|
||||
except (ValueError, OSError):
|
||||
return False
|
||||
|
||||
|
||||
def _skill_of(root: Path, path: Path) -> tuple[str, str, str]:
|
||||
"""Return ``(category, skill_name, posix_rel_path)`` for a scanned file."""
|
||||
|
||||
rel = path.relative_to(root)
|
||||
parts = rel.parts
|
||||
category = parts[0] if parts else ""
|
||||
skill_name = parts[1] if len(parts) > 1 else ""
|
||||
return category, skill_name, rel.as_posix()
|
||||
|
||||
|
||||
def _read_text(path: Path) -> str | None:
|
||||
"""Read UTF-8 text preserving original line endings; skip binary/bad files."""
|
||||
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8", newline="") as fh:
|
||||
return fh.read()
|
||||
except (UnicodeDecodeError, OSError):
|
||||
return None
|
||||
|
||||
|
||||
def _write_text(path: Path, text: str) -> None:
|
||||
with open(path, "w", encoding="utf-8", newline="") as fh:
|
||||
fh.write(text)
|
||||
|
||||
|
||||
# ── synchronous workers (run via asyncio.to_thread; never block the loop) ─────
|
||||
|
||||
|
||||
def _iter_skill_files(root: Path, skill: str | None = None) -> list[Path]:
|
||||
"""All ``*.md`` / ``*.py`` under ``{root}/{public,custom}``.
|
||||
|
||||
Hidden directories (``.address-edits`` backups, ``.history``) are skipped.
|
||||
When *skill* is given, only that skill's directory is walked.
|
||||
"""
|
||||
|
||||
files: list[Path] = []
|
||||
for sub in _SCAN_SUBDIRS:
|
||||
base = root / sub
|
||||
if skill:
|
||||
base = base / skill
|
||||
if not base.is_dir():
|
||||
continue
|
||||
for p in base.rglob("*"):
|
||||
if not p.is_file() or p.suffix.lower() not in _SCAN_EXTS:
|
||||
continue
|
||||
try:
|
||||
rel_parts = p.relative_to(root).parts
|
||||
except ValueError:
|
||||
continue
|
||||
if any(part.startswith(".") for part in rel_parts):
|
||||
continue
|
||||
files.append(p)
|
||||
return files
|
||||
|
||||
|
||||
def _group_files_by_skill(root: Path, files: list[Path]) -> dict[tuple[str, str], list[Path]]:
|
||||
"""Bucket scanned files under their owning ``(category, skill_name)``."""
|
||||
|
||||
buckets: dict[tuple[str, str], list[Path]] = {}
|
||||
for p in files:
|
||||
category, skill_name, _ = _skill_of(root, p)
|
||||
buckets.setdefault((category, skill_name), []).append(p)
|
||||
return buckets
|
||||
|
||||
|
||||
def _occurrence_kind(value: str) -> str:
|
||||
return infer_kind(value) or "url"
|
||||
|
||||
|
||||
def _merge_occurrences(groups: dict[str, AddressGroup], pairs: list[tuple[str, OccurrenceModel]]) -> None:
|
||||
"""Merge ``(address, occurrence)`` pairs into the aggregate group map."""
|
||||
|
||||
for address, occ in pairs:
|
||||
g = groups.get(address)
|
||||
if g is None:
|
||||
g = AddressGroup(address=address, kind=_occurrence_kind(address), count=0, occurrences=[])
|
||||
groups[address] = g
|
||||
g.count += 1
|
||||
g.occurrences.append(occ)
|
||||
|
||||
|
||||
def _plan_apply(root: Path, mapping: dict[str, str]) -> tuple[list[tuple[Path, str, int, str]], int]:
|
||||
"""Compute the rewrite plan without touching disk → (plan, current_total)."""
|
||||
|
||||
plan: list[tuple[Path, str, int, str]] = []
|
||||
current_total = 0
|
||||
for path in _iter_skill_files(root):
|
||||
text = _read_text(path)
|
||||
if text is None:
|
||||
continue
|
||||
new_text, cnt = apply_replacements(text, mapping, VALID_KINDS)
|
||||
if cnt <= 0:
|
||||
continue
|
||||
if not _is_within(root, path):
|
||||
raise HTTPException(status_code=400, detail=f"拒绝写入技能目录之外的文件:{path}")
|
||||
_, _, rel = _skill_of(root, path)
|
||||
plan.append((path, new_text, cnt, rel))
|
||||
current_total += cnt
|
||||
return plan, current_total
|
||||
|
||||
|
||||
def _backup_file(root: Path, backup_id: str, path: Path, rel: str) -> None:
|
||||
dest = root / _BACKUP_DIRNAME / backup_id / rel
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(path, dest)
|
||||
|
||||
|
||||
# ── scan extraction shared by one-shot + stream ───────────────────────────────
|
||||
|
||||
|
||||
def _scan_files_with_addresses(root: Path, files: list[Path], wanted: tuple[str, ...]) -> tuple[list[tuple[str, OccurrenceModel]], int]:
|
||||
"""Scan files → ([(address, occurrence)], files_read)."""
|
||||
|
||||
pairs: list[tuple[str, OccurrenceModel]] = []
|
||||
read = 0
|
||||
for path in files:
|
||||
text = _read_text(path)
|
||||
if text is None:
|
||||
continue
|
||||
read += 1
|
||||
category, skill_name, rel = _skill_of(root, path)
|
||||
for o in scan_occurrences(text, wanted):
|
||||
pairs.append(
|
||||
(
|
||||
o.value,
|
||||
OccurrenceModel(skill=skill_name, category=category, file=rel, line=o.line, col=o.col, snippet=o.snippet),
|
||||
)
|
||||
)
|
||||
return pairs, read
|
||||
|
||||
|
||||
def _ordered_groups(groups: dict[str, AddressGroup]) -> list[AddressGroup]:
|
||||
return sorted(groups.values(), key=lambda g: (-g.count, g.address))
|
||||
|
||||
|
||||
def _build_mapping(replacements: list[Replacement]) -> tuple[dict[str, str], list[tuple[str, str, str]], list[str]]:
|
||||
"""Validate replacements → ``(mapping, [(from,to,kind)], warnings)``.
|
||||
|
||||
Drops (with a warning) any entry that is empty, duplicated, not a recognized
|
||||
URL/IP, a no-op (``from == to``), or whose ``to`` is malformed for its kind.
|
||||
"""
|
||||
|
||||
mapping: dict[str, str] = {}
|
||||
norm: list[tuple[str, str, str]] = []
|
||||
warnings: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for r in replacements:
|
||||
frm = r.from_.strip()
|
||||
to = r.to.strip()
|
||||
if not frm:
|
||||
continue
|
||||
if frm in seen:
|
||||
warnings.append(f"重复的原始地址已忽略:{frm}")
|
||||
continue
|
||||
kind = (r.kind or "").strip().lower() or infer_kind(frm)
|
||||
if kind not in VALID_KINDS:
|
||||
warnings.append(f"无法识别为 URL/IP,已忽略:{frm}")
|
||||
continue
|
||||
if to == frm:
|
||||
warnings.append(f"替换值与原值相同,已忽略:{frm}")
|
||||
continue
|
||||
if not is_valid_replacement(kind, to):
|
||||
warnings.append(f"替换值不是合法的 {kind},已忽略:{to}")
|
||||
continue
|
||||
seen.add(frm)
|
||||
mapping[frm] = to
|
||||
norm.append((frm, to, kind))
|
||||
return mapping, norm, warnings
|
||||
|
||||
|
||||
def _ndjson(obj: dict) -> bytes:
|
||||
return (json.dumps(obj, ensure_ascii=False) + "\n").encode("utf-8")
|
||||
|
||||
|
||||
# ── routes: scan ─────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@router.get("/scan", response_model=ScanResponse)
|
||||
async def scan_addresses(
|
||||
request: Request,
|
||||
kinds: str | None = Query(default=None, description="逗号分隔:url,ip(默认全部)"),
|
||||
skill: str | None = Query(default=None, description="仅扫描指定技能目录"),
|
||||
) -> ScanResponse:
|
||||
await _require_admin(request)
|
||||
root = _skills_root()
|
||||
wanted = normalize_kinds(kinds)
|
||||
files = await asyncio.to_thread(_iter_skill_files, root, skill)
|
||||
buckets = _group_files_by_skill(root, files)
|
||||
groups: dict[str, AddressGroup] = {}
|
||||
scanned = 0
|
||||
for paths in buckets.values():
|
||||
pairs, read = await asyncio.to_thread(_scan_files_with_addresses, root, paths, wanted)
|
||||
_merge_occurrences(groups, pairs)
|
||||
scanned += read
|
||||
return ScanResponse(
|
||||
groups=_ordered_groups(groups),
|
||||
scanned_files=scanned,
|
||||
scanned_skills=len(buckets),
|
||||
kinds=list(wanted),
|
||||
scanned_at=datetime.now(UTC).isoformat(),
|
||||
)
|
||||
|
||||
|
||||
@router.get("/scan/stream")
|
||||
async def scan_addresses_stream(
|
||||
request: Request,
|
||||
kinds: str | None = Query(default=None),
|
||||
skill: str | None = Query(default=None),
|
||||
) -> StreamingResponse:
|
||||
"""Scan with per-skill NDJSON progress.
|
||||
|
||||
Emits ``{"type":"start","total":N}`` then one
|
||||
``{"type":"progress","done":i,"total":N,"skill":name}`` per skill, finally
|
||||
``{"type":"result", ...ScanResponse...}``.
|
||||
"""
|
||||
|
||||
await _require_admin(request)
|
||||
root = _skills_root()
|
||||
wanted = normalize_kinds(kinds)
|
||||
|
||||
async def gen() -> AsyncIterator[bytes]:
|
||||
files = await asyncio.to_thread(_iter_skill_files, root, skill)
|
||||
buckets = _group_files_by_skill(root, files)
|
||||
total = len(buckets)
|
||||
yield _ndjson({"type": "start", "total": total})
|
||||
groups: dict[str, AddressGroup] = {}
|
||||
scanned = 0
|
||||
done = 0
|
||||
for (category, skill_name), paths in buckets.items():
|
||||
pairs, read = await asyncio.to_thread(_scan_files_with_addresses, root, paths, wanted)
|
||||
_merge_occurrences(groups, pairs)
|
||||
scanned += read
|
||||
done += 1
|
||||
yield _ndjson({"type": "progress", "done": done, "total": total, "skill": skill_name or category})
|
||||
result = ScanResponse(
|
||||
groups=_ordered_groups(groups),
|
||||
scanned_files=scanned,
|
||||
scanned_skills=total,
|
||||
kinds=list(wanted),
|
||||
scanned_at=datetime.now(UTC).isoformat(),
|
||||
)
|
||||
yield _ndjson({"type": "result", **result.model_dump()})
|
||||
|
||||
return StreamingResponse(gen(), media_type="application/x-ndjson")
|
||||
|
||||
|
||||
# ── routes: preview ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _compute_preview(root: Path, mapping: dict[str, str], norm: list[tuple[str, str, str]]) -> tuple[list[PreviewItem], int]:
|
||||
counts: dict[str, int] = {frm: 0 for frm, _, _ in norm}
|
||||
files: dict[str, list[PreviewFileHit]] = {frm: [] for frm, _, _ in norm}
|
||||
total = 0
|
||||
if mapping:
|
||||
for path in _iter_skill_files(root):
|
||||
text = _read_text(path)
|
||||
if text is None:
|
||||
continue
|
||||
_, _, rel = _skill_of(root, path)
|
||||
for o in scan_occurrences(text, VALID_KINDS):
|
||||
if o.value not in mapping:
|
||||
continue
|
||||
to = mapping[o.value]
|
||||
counts[o.value] += 1
|
||||
total += 1
|
||||
files[o.value].append(PreviewFileHit(file=rel, line=o.line, before=o.snippet, after=o.snippet.replace(o.value, to)))
|
||||
items = [PreviewItem(from_=frm, to=to, kind=kind, count=counts[frm], files=files[frm]) for frm, to, kind in norm]
|
||||
return items, total
|
||||
|
||||
|
||||
@router.post("/preview", response_model=PreviewResponse)
|
||||
async def preview_replacements(request: Request, body: PreviewRequest) -> PreviewResponse:
|
||||
await _require_admin(request)
|
||||
root = _skills_root()
|
||||
mapping, norm, warnings = _build_mapping(body.replacements)
|
||||
items, total = await asyncio.to_thread(_compute_preview, root, mapping, norm)
|
||||
return PreviewResponse(items=items, total=total, warnings=warnings, expected_total=total)
|
||||
|
||||
|
||||
# ── routes: apply ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _new_backup_id() -> str:
|
||||
return datetime.now(UTC).strftime("%Y%m%d-%H%M%S-%f")
|
||||
|
||||
|
||||
@router.post("/apply", response_model=ApplyResponse)
|
||||
async def apply_replacements_route(request: Request, body: ApplyRequest) -> ApplyResponse:
|
||||
await _require_admin(request)
|
||||
root = _skills_root()
|
||||
mapping, _norm, warnings = _build_mapping(body.replacements)
|
||||
if not mapping:
|
||||
return ApplyResponse(changed_files=[], replaced_total=0, backup_id=None, warnings=warnings)
|
||||
|
||||
plan, current_total = await asyncio.to_thread(_plan_apply, root, mapping)
|
||||
if body.expected_total is not None and body.expected_total != current_total:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail=f"文件自上次扫描后已变化(预期 {body.expected_total} 处,现为 {current_total} 处),请重新扫描后再应用。",
|
||||
)
|
||||
if not plan:
|
||||
return ApplyResponse(changed_files=[], replaced_total=0, backup_id=None, warnings=warnings)
|
||||
|
||||
backup_id: str | None = None
|
||||
if body.backup:
|
||||
backup_id = _new_backup_id()
|
||||
for path, _new, _cnt, rel in plan:
|
||||
await asyncio.to_thread(_backup_file, root, backup_id, path, rel)
|
||||
|
||||
changed: list[ChangedFile] = []
|
||||
replaced_total = 0
|
||||
for path, new_text, cnt, rel in plan:
|
||||
await asyncio.to_thread(_write_text, path, new_text)
|
||||
changed.append(ChangedFile(file=rel, count=cnt))
|
||||
replaced_total += cnt
|
||||
|
||||
await _refresh_cache_safe()
|
||||
logger.info("Skill address apply: %d files, %d replacements, backup=%s", len(changed), replaced_total, backup_id)
|
||||
return ApplyResponse(changed_files=changed, replaced_total=replaced_total, backup_id=backup_id, warnings=warnings)
|
||||
|
||||
|
||||
@router.post("/apply/stream")
|
||||
async def apply_replacements_stream(request: Request, body: ApplyRequest) -> StreamingResponse:
|
||||
"""Apply with per-file NDJSON progress.
|
||||
|
||||
Validation + the optimistic-lock check happen *before* streaming (so a 409
|
||||
is a normal HTTP error). Then: ``{"type":"start","total_files":M,...}`` →
|
||||
one ``{"type":"progress","done":k,...}`` per written file →
|
||||
``{"type":"result", ...ApplyResponse...}``.
|
||||
"""
|
||||
|
||||
await _require_admin(request)
|
||||
root = _skills_root()
|
||||
mapping, _norm, warnings = _build_mapping(body.replacements)
|
||||
|
||||
plan: list[tuple[Path, str, int, str]] = []
|
||||
current_total = 0
|
||||
if mapping:
|
||||
plan, current_total = await asyncio.to_thread(_plan_apply, root, mapping)
|
||||
if body.expected_total is not None and body.expected_total != current_total:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail=f"文件自上次扫描后已变化(预期 {body.expected_total} 处,现为 {current_total} 处),请重新扫描后再应用。",
|
||||
)
|
||||
|
||||
async def gen() -> AsyncIterator[bytes]:
|
||||
if not plan:
|
||||
result = ApplyResponse(changed_files=[], replaced_total=0, backup_id=None, warnings=warnings)
|
||||
yield _ndjson({"type": "result", **result.model_dump()})
|
||||
return
|
||||
|
||||
backup_id: str | None = None
|
||||
if body.backup:
|
||||
backup_id = _new_backup_id()
|
||||
|
||||
yield _ndjson({"type": "start", "total_files": len(plan), "total_replacements": current_total})
|
||||
|
||||
changed: list[ChangedFile] = []
|
||||
replaced_total = 0
|
||||
done = 0
|
||||
for path, new_text, cnt, rel in plan:
|
||||
if backup_id:
|
||||
await asyncio.to_thread(_backup_file, root, backup_id, path, rel)
|
||||
await asyncio.to_thread(_write_text, path, new_text)
|
||||
changed.append(ChangedFile(file=rel, count=cnt))
|
||||
replaced_total += cnt
|
||||
done += 1
|
||||
yield _ndjson({"type": "progress", "done": done, "total_files": len(plan), "file": rel, "count": cnt})
|
||||
|
||||
await _refresh_cache_safe()
|
||||
logger.info("Skill address apply(stream): %d files, %d replacements, backup=%s", len(changed), replaced_total, backup_id)
|
||||
result = ApplyResponse(changed_files=changed, replaced_total=replaced_total, backup_id=backup_id, warnings=warnings)
|
||||
yield _ndjson({"type": "result", **result.model_dump()})
|
||||
|
||||
return StreamingResponse(gen(), media_type="application/x-ndjson")
|
||||
|
||||
|
||||
# ── routes: backups / rollback ───────────────────────────────────────────────
|
||||
|
||||
|
||||
def _list_backups(root: Path) -> list[BackupEntry]:
|
||||
backup_root = root / _BACKUP_DIRNAME
|
||||
entries: list[BackupEntry] = []
|
||||
if backup_root.is_dir():
|
||||
for d in backup_root.iterdir():
|
||||
if not d.is_dir():
|
||||
continue
|
||||
file_count = sum(1 for p in d.rglob("*") if p.is_file())
|
||||
try:
|
||||
created = datetime.fromtimestamp(d.stat().st_mtime, UTC).isoformat()
|
||||
except OSError:
|
||||
created = ""
|
||||
entries.append(BackupEntry(backup_id=d.name, files=file_count, created_at=created))
|
||||
entries.sort(key=lambda e: e.backup_id, reverse=True)
|
||||
return entries
|
||||
|
||||
|
||||
@router.get("/backups", response_model=BackupListResponse)
|
||||
async def list_backups(request: Request) -> BackupListResponse:
|
||||
await _require_admin(request)
|
||||
root = _skills_root()
|
||||
entries = await asyncio.to_thread(_list_backups, root)
|
||||
return BackupListResponse(backups=entries)
|
||||
|
||||
|
||||
def _restore_backup(root: Path, backup_root: Path) -> list[str]:
|
||||
restored: list[str] = []
|
||||
for src in backup_root.rglob("*"):
|
||||
if not src.is_file():
|
||||
continue
|
||||
rel = src.relative_to(backup_root)
|
||||
dest = root / rel
|
||||
if not _is_within(root, dest):
|
||||
continue
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(src, dest)
|
||||
restored.append(rel.as_posix())
|
||||
return restored
|
||||
|
||||
|
||||
@router.post("/rollback/{backup_id}", response_model=RollbackResponse)
|
||||
async def rollback(request: Request, backup_id: str) -> RollbackResponse:
|
||||
await _require_admin(request)
|
||||
root = _skills_root()
|
||||
if "/" in backup_id or "\\" in backup_id or backup_id in (".", ".."):
|
||||
raise HTTPException(status_code=400, detail="非法的备份编号")
|
||||
backup_root = (root / _BACKUP_DIRNAME / backup_id).resolve()
|
||||
if not _is_within(root, backup_root) or not backup_root.is_dir():
|
||||
raise HTTPException(status_code=404, detail="备份不存在")
|
||||
|
||||
restored = await asyncio.to_thread(_restore_backup, root, backup_root)
|
||||
await _refresh_cache_safe()
|
||||
logger.info("Skill address rollback: backup=%s, restored %d files", backup_id, len(restored))
|
||||
return RollbackResponse(backup_id=backup_id, restored_files=restored)
|
||||
|
||||
|
||||
# ── misc ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def _refresh_cache_safe() -> None:
|
||||
try:
|
||||
await refresh_skills_system_prompt_cache_async()
|
||||
except Exception: # noqa: BLE001 — cache refresh is best-effort
|
||||
logger.warning("Failed to refresh skills prompt cache", exc_info=True)
|
||||
Reference in new issue
Block a user