feat(skills): 自我进化后端 — curator 配置可编辑 + 运行日志可读
后端骨架： - curator.py / usage.py / skill_tools.py：基于使用计数的生命周期管理 + LLM 合并/归档建议（同事已有原型，集中提交） - skill_evolution_config.py：CuratorConfig 嵌套模型 - app.py：启动时按间隔触发 curator 后台线程 - tools.py：开启自我进化时一并注册 skill_list / skill_view 工具 - skill_manage_tool.py：agent 创建技能时标记 source=agent 本次新增： - skill_evolution_runtime.py：skill_evolution.json 读写，原子写入 - app_config.py：把 overrides 深合并到 AppConfig.skill_evolution， 并把 overrides 的 mtime 纳入缓存失效条件，前端 PUT 后热生效 - curator_log.py：进程内 ring buffer (maxlen=200)，所有 record/get 操作内部兜异常，保证 logging 永远不会让 curator 跑挂 - curator.py 关键节点插桩：run_started / 自动转换计数 / 候选数 / LLM agent invoke / 解析结果 / archive 动作 / run complete - routers/skills.py：GET/PUT /api/curator/config（写到 skill_evolution.json + reset_app_config）+ GET /api/curator/logs 测试： - tests/test_skill_evolution_runtime.py：overrides 往返、损坏 JSON 容错、AppConfig 合并、ring buffer 线程安全、record 异常吞咽 杂项： - .gitignore：忽略 .deer-flow/* 与 skills/* 等运行时目录 - uv.lock：跟随依赖锁更新 - 清理已废弃的 agent runtime 数据 SOUL.md Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
xuhdandClaude Opus 4.7 committed
1 parent
230eec1baf
commit
9b6b94d919
14 files changed
+3981
-2698
No files matched your search
@@ -1,9 +1,10 @@
|
||||
import json
|
||||
import logging
|
||||
import tempfile
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
|
||||
from fastapi import APIRouter, Body, Depends, File, HTTPException, UploadFile
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.gateway.deps import get_config
|
||||
@@ -30,6 +31,13 @@ class SkillResponse(BaseModel):
|
||||
license: str | None = Field(None, description="License information")
|
||||
category: SkillCategory = Field(..., description="Category of the skill (public or custom)")
|
||||
enabled: bool = Field(default=True, description="Whether this skill is enabled")
|
||||
use_count: int = Field(default=0, description="Number of times the skill has been used")
|
||||
view_count: int = Field(default=0, description="Number of times the skill has been viewed")
|
||||
patch_count: int = Field(default=0, description="Number of times the skill has been patched")
|
||||
last_activity_at: datetime | None = Field(default=None, description="Most recent activity timestamp")
|
||||
state: str = Field(default="active", description="Lifecycle state: active, stale, or archived")
|
||||
pinned: bool = Field(default=False, description="Whether this skill is pinned (excluded from curation)")
|
||||
source: str = Field(default="upload", description="Origin: 'agent' for agent-created, 'upload' for user-uploaded")
|
||||
|
||||
|
||||
class SkillsListResponse(BaseModel):
|
||||
@@ -91,12 +99,29 @@ class SkillRollbackRequest(BaseModel):
|
||||
|
||||
def _skill_to_response(skill: Skill) -> SkillResponse:
|
||||
"""Convert a Skill object to a SkillResponse."""
|
||||
from deerflow.skills.usage import get_entry, latest_activity_at
|
||||
|
||||
entry = get_entry(skill.name)
|
||||
last_at_str = latest_activity_at(skill.name)
|
||||
last_at: datetime | None = None
|
||||
if last_at_str:
|
||||
try:
|
||||
last_at = datetime.fromisoformat(last_at_str)
|
||||
except Exception:
|
||||
pass
|
||||
return SkillResponse(
|
||||
name=skill.name,
|
||||
description=skill.description,
|
||||
license=skill.license,
|
||||
category=skill.category,
|
||||
enabled=skill.enabled,
|
||||
use_count=entry.get("use_count", 0),
|
||||
view_count=entry.get("view_count", 0),
|
||||
patch_count=entry.get("patch_count", 0),
|
||||
last_activity_at=last_at,
|
||||
state=entry.get("state", "active"),
|
||||
pinned=entry.get("pinned", False),
|
||||
source=entry.get("source", "upload"),
|
||||
)
|
||||
|
||||
|
||||
@@ -202,6 +227,11 @@ async def install_skill_upload(
|
||||
try:
|
||||
storage = get_or_new_skill_storage(app_config=config)
|
||||
result = await storage.ainstall_skill_from_archive(tmp_path, overwrite=overwrite, require_skill_suffix=False)
|
||||
try:
|
||||
from deerflow.skills.usage import mark_uploaded
|
||||
mark_uploaded(result["skill_name"])
|
||||
except Exception:
|
||||
pass
|
||||
await refresh_skills_system_prompt_cache_async()
|
||||
return SkillInstallResponse(**result)
|
||||
except FileNotFoundError as e:
|
||||
@@ -229,6 +259,11 @@ async def install_skill(request: SkillInstallRequest, config: AppConfig = Depend
|
||||
try:
|
||||
skill_file_path = resolve_thread_virtual_path(request.thread_id, request.path)
|
||||
result = await get_or_new_skill_storage(app_config=config).ainstall_skill_from_archive(skill_file_path)
|
||||
try:
|
||||
from deerflow.skills.usage import mark_uploaded
|
||||
mark_uploaded(result["skill_name"])
|
||||
except Exception:
|
||||
pass
|
||||
await refresh_skills_system_prompt_cache_async()
|
||||
return SkillInstallResponse(**result)
|
||||
except FileNotFoundError as e:
|
||||
@@ -294,6 +329,11 @@ async def update_custom_skill(skill_name: str, request: CustomSkillUpdateRequest
|
||||
"scanner": {"decision": scan.decision, "reason": scan.reason},
|
||||
},
|
||||
)
|
||||
try:
|
||||
from deerflow.skills.usage import bump_patch
|
||||
bump_patch(skill_name)
|
||||
except Exception:
|
||||
pass
|
||||
await refresh_skills_system_prompt_cache_async()
|
||||
return await get_custom_skill(skill_name, config)
|
||||
except HTTPException:
|
||||
@@ -324,6 +364,11 @@ async def delete_custom_skill(skill_name: str, config: AppConfig = Depends(get_c
|
||||
"scanner": {"decision": "allow", "reason": "Deletion requested."},
|
||||
},
|
||||
)
|
||||
try:
|
||||
from deerflow.skills.usage import forget
|
||||
forget(skill_name)
|
||||
except Exception:
|
||||
pass
|
||||
await refresh_skills_system_prompt_cache_async()
|
||||
return {"success": True}
|
||||
except FileNotFoundError as e:
|
||||
@@ -469,3 +514,168 @@ async def update_skill(skill_name: str, request: SkillUpdateRequest, config: App
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to update skill {skill_name}: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=f"Failed to update skill: {str(e)}")
|
||||
|
||||
|
||||
class SkillPinRequest(BaseModel):
|
||||
pinned: bool = Field(..., description="Whether to pin or unpin the skill")
|
||||
|
||||
|
||||
@router.post("/skills/custom/{skill_name}/pin", response_model=SkillResponse, summary="Pin or Unpin Skill")
|
||||
async def pin_skill(skill_name: str, request: SkillPinRequest, config: AppConfig = Depends(get_config)) -> SkillResponse:
|
||||
"""Pin a skill to prevent it from being automatically archived by the curator."""
|
||||
try:
|
||||
skill_name = skill_name.replace("\r\n", "").replace("\n", "")
|
||||
storage = get_or_new_skill_storage(app_config=config)
|
||||
if not storage.custom_skill_exists(skill_name):
|
||||
raise HTTPException(status_code=404, detail=f"Custom skill '{skill_name}' not found")
|
||||
from deerflow.skills.usage import set_pinned
|
||||
set_pinned(skill_name, request.pinned)
|
||||
skills = storage.load_skills(enabled_only=False)
|
||||
skill = next((s for s in skills if s.name == skill_name), None)
|
||||
if skill is None:
|
||||
raise HTTPException(status_code=404, detail=f"Skill '{skill_name}' not found")
|
||||
return _skill_to_response(skill)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error("Failed to pin skill %s: %s", skill_name, e, exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=f"Failed to pin skill: {str(e)}")
|
||||
|
||||
|
||||
@router.get("/curator/status", summary="Curator Status")
|
||||
async def curator_status() -> dict:
|
||||
"""Return the current curator state and per-lifecycle skill counts."""
|
||||
try:
|
||||
from deerflow.skills.curator import get_curator_status
|
||||
return get_curator_status()
|
||||
except Exception as e:
|
||||
logger.error("Failed to get curator status: %s", e, exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=f"Failed to get curator status: {str(e)}")
|
||||
|
||||
|
||||
@router.post("/curator/run", summary="Trigger Curator Run")
|
||||
async def run_curator() -> dict:
|
||||
"""Manually trigger an immediate curator review pass (bypasses interval check, runs in background thread)."""
|
||||
try:
|
||||
import threading
|
||||
from deerflow.skills.curator import _curator_thread_target
|
||||
thread = threading.Thread(target=_curator_thread_target, daemon=True, name="skill-curator-manual")
|
||||
thread.start()
|
||||
return {"triggered": True, "message": "Curator run started in background"}
|
||||
except Exception as e:
|
||||
logger.error("Failed to trigger curator: %s", e, exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=f"Failed to trigger curator: {str(e)}")
|
||||
|
||||
|
||||
class CuratorSubConfig(BaseModel):
|
||||
"""Nested curator configuration mirroring CuratorConfig."""
|
||||
|
||||
enabled: bool = Field(default=True, description="Whether the curator runs automatically.")
|
||||
interval_hours: int = Field(default=168, ge=1, description="Hours between curator runs.")
|
||||
stale_after_days: int = Field(default=30, ge=1, description="Days of inactivity before a skill is marked stale.")
|
||||
archive_after_days: int = Field(default=90, ge=1, description="Days of inactivity before a stale skill is archived.")
|
||||
|
||||
|
||||
class CuratorConfigResponse(BaseModel):
|
||||
"""Response model exposing the effective skill_evolution config."""
|
||||
|
||||
enabled: bool
|
||||
moderation_model_name: str | None
|
||||
curator: CuratorSubConfig
|
||||
|
||||
|
||||
class CuratorConfigRequest(BaseModel):
|
||||
"""Payload for updating skill_evolution overrides.
|
||||
|
||||
All fields are required so the frontend always sends a complete config;
|
||||
backend writes the entire payload to skill_evolution.json atomically.
|
||||
"""
|
||||
|
||||
enabled: bool = Field(..., description="Whether the agent can create and modify skills under skills/custom.")
|
||||
moderation_model_name: str | None = Field(default=None, description="Optional model name for skill security moderation.")
|
||||
curator: CuratorSubConfig
|
||||
|
||||
|
||||
class CuratorLogRecord(BaseModel):
|
||||
ts: str
|
||||
level: str
|
||||
message: str
|
||||
|
||||
|
||||
class CuratorLogsResponse(BaseModel):
|
||||
records: list[CuratorLogRecord]
|
||||
run_count: int
|
||||
last_run_at: str | None
|
||||
|
||||
|
||||
@router.get("/curator/config", response_model=CuratorConfigResponse, summary="Get Skill Evolution Config")
|
||||
async def get_curator_config(config: AppConfig = Depends(get_config)) -> CuratorConfigResponse:
|
||||
"""Return the current effective skill_evolution config (yaml defaults + skill_evolution.json overrides)."""
|
||||
try:
|
||||
evo = config.skill_evolution
|
||||
return CuratorConfigResponse(
|
||||
enabled=evo.enabled,
|
||||
moderation_model_name=evo.moderation_model_name,
|
||||
curator=CuratorSubConfig(
|
||||
enabled=evo.curator.enabled,
|
||||
interval_hours=evo.curator.interval_hours,
|
||||
stale_after_days=evo.curator.stale_after_days,
|
||||
archive_after_days=evo.curator.archive_after_days,
|
||||
),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error("Failed to read curator config: %s", e, exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=f"Failed to read curator config: {str(e)}")
|
||||
|
||||
|
||||
@router.put("/curator/config", response_model=CuratorConfigResponse, summary="Update Skill Evolution Config")
|
||||
async def update_curator_config(body: CuratorConfigRequest) -> CuratorConfigResponse:
|
||||
"""Persist overrides to skill_evolution.json and force AppConfig to reload on next read.
|
||||
|
||||
The payload is validated against the SkillEvolutionConfig shape; unknown
|
||||
fields are rejected by the Pydantic model_validate call.
|
||||
"""
|
||||
try:
|
||||
from deerflow.config.skill_evolution_config import SkillEvolutionConfig
|
||||
from deerflow.config.skill_evolution_runtime import save_overrides
|
||||
|
||||
payload = body.model_dump()
|
||||
# Round-trip through SkillEvolutionConfig to ensure persisted shape stays valid.
|
||||
SkillEvolutionConfig.model_validate(payload)
|
||||
save_overrides(payload)
|
||||
|
||||
# Force the next get_app_config() call to pick up the new overrides.
|
||||
# mtime check inside get_app_config() will trigger reload automatically,
|
||||
# but resetting here makes the response below reflect the new state.
|
||||
from deerflow.config.app_config import reset_app_config
|
||||
reset_app_config()
|
||||
|
||||
return await get_curator_config()
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error("Failed to update curator config: %s", e, exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=f"Failed to update curator config: {str(e)}")
|
||||
|
||||
|
||||
@router.get("/curator/logs", response_model=CuratorLogsResponse, summary="Recent Curator Logs")
|
||||
async def get_curator_logs() -> CuratorLogsResponse:
|
||||
"""Return in-memory log records from the most recent curator run.
|
||||
|
||||
The buffer is process-local and cleared when curator starts a new run or
|
||||
when the backend restarts. Records are ordered oldest → newest.
|
||||
"""
|
||||
try:
|
||||
from deerflow.skills import curator_log
|
||||
from deerflow.skills.curator import get_curator_status
|
||||
|
||||
records = curator_log.get_recent()
|
||||
status = get_curator_status()
|
||||
return CuratorLogsResponse(
|
||||
records=[CuratorLogRecord(**r) for r in records],
|
||||
run_count=int(status.get("run_count", 0)),
|
||||
last_run_at=status.get("last_run_at"),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error("Failed to read curator logs: %s", e, exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=f"Failed to read curator logs: {str(e)}")
|
||||
Reference in new issue
Block a user