Merge remote-tracking branch 'origin/master' into ji
This commit is contained in:
commit
4761677676
74 files changed
+9770
-2887
No files matched your search
@@ -302,6 +302,55 @@ Custom agent metadata lives in the `agents` ORM table (`deerflow.persistence.age
|
||||
- **Injection**: Enabled skills listed in agent system prompt with container paths
|
||||
- **Installation**: `POST /api/skills/install` extracts .skill ZIP archive to custom/ directory
|
||||
|
||||
#### Skill Curator (`packages/harness/deerflow/skills/curator.py`)
|
||||
|
||||
Background skill lifecycle manager. The Gateway `lifespan` starts a resident
|
||||
scheduler task (`_curator_scheduler_loop` in `app/gateway/app.py`) that — after a
|
||||
5-minute startup delay — calls `maybe_run_curator()` every hour. A cross-process
|
||||
file lock (`.curator_scheduler.lock`) elects a single leader worker so the
|
||||
scheduler runs once per deployment. Per run:
|
||||
|
||||
- **Master switch** — `skill_evolution.enabled` (default **`false`**) gates the
|
||||
whole feature; `curator.enabled` (default **`false`**) gates the curator
|
||||
specifically. Both must be `true` for the curator to act. `should_run_now()`
|
||||
and `_curator_thread_target()` both enforce this — so manual `POST
|
||||
/api/curator/run` also respects the master switch. Config is
|
||||
`config.yaml.skill_evolution` deep-merged with per-user
|
||||
`users/{user_id}/skill_evolution.json`.
|
||||
- **Per-user run lock** — `.curator_state` carries `running` / `started_at`;
|
||||
`_try_acquire_run_lock()` prevents the same user's curator from running
|
||||
concurrently (scheduler vs. manual trigger, or double manual clicks). A lock
|
||||
whose `started_at` is older than `_CURATOR_STALE_LOCK_HOURS` (6h) is treated as
|
||||
process-crash residue and can be taken over.
|
||||
- **Concurrency cap** — `maybe_run_curator()` collects all due users and hands
|
||||
them to `_dispatch_curator_runs()`, which runs them through a
|
||||
`ThreadPoolExecutor` capped at `_CURATOR_MAX_CONCURRENCY` (5) — avoids a
|
||||
thundering herd of LLM calls when many users come due at once (e.g. first
|
||||
deploy).
|
||||
- **Failure back-off** — `_release_run_lock()` always stamps `last_attempt_at`.
|
||||
When a first run keeps crashing (`last_run_at` never set), `should_run_now()`
|
||||
retries on `_CURATOR_RETRY_HOURS` (6h) instead of every hourly tick.
|
||||
- **`should_run_now()`** is the real gate: the hourly tick only checks "is the
|
||||
per-user `interval_hours` (default 168h) elapsed". Timestamp parsing tolerates
|
||||
naive (tz-less) strings — a corrupt `last_run_at` triggers a re-run rather than
|
||||
a permanent deadlock.
|
||||
- **Published-skill filter** — `SkillStore.list_published_names()` is a batch
|
||||
query; the curator fetches the published-name set once per run instead of one
|
||||
DB round-trip per candidate skill.
|
||||
- **LLM review timeout** — `_run_llm_review()` runs `agent.invoke()` in a worker
|
||||
thread bounded by `curator.review_timeout_seconds` (default 600s). The model's
|
||||
per-request `request_timeout` can't bound a multi-turn React agent loop, so on
|
||||
timeout the run is abandoned (a warning is written to `curator_log`, visible in
|
||||
the Web UI) and the curator finishes cleanly + releases the run lock. The
|
||||
worker context is copied via `contextvars.copy_context()` so tool calls still
|
||||
resolve the correct `user_id`.
|
||||
- **Review model** — `_run_llm_review()` uses `curator.model_name` (a
|
||||
`config.yaml` `models[]` entry name) if set, else the primary model; it is
|
||||
editable from the Web UI evolution panel. The skill security scanner
|
||||
(`scan_skill_content` in `security_scanner.py`) reuses the same
|
||||
`skill_evolution.curator.model_name` setting — review and moderation share one
|
||||
model. There is no separate moderation-model config field.
|
||||
|
||||
### Model Factory (`packages/harness/deerflow/models/factory.py`)
|
||||
|
||||
- `create_chat_model(name, thinking_enabled)` instantiates LLM from config via reflection
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from collections.abc import AsyncGenerator
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
@@ -268,6 +269,93 @@ async def _run_db_migrations() -> None:
|
||||
logger.exception("Database migration failed — schema may be out of date")
|
||||
|
||||
|
||||
# Skill curator 调度参数。
|
||||
# - 检查间隔:每隔这么久检查一次"需不需要进化";真正是否执行由 should_run_now() 判定。
|
||||
# - 启动延迟:启动后先等这么久再首次检查,避开启动高峰。
|
||||
_CURATOR_CHECK_INTERVAL_SECONDS = 3600 # 1 小时
|
||||
_CURATOR_STARTUP_DELAY_SECONDS = 300 # 5 分钟
|
||||
|
||||
|
||||
def _acquire_singleton_lock(lock_path):
|
||||
"""跨平台非阻塞独占文件锁,用于多 worker 下的 leader 选举。
|
||||
|
||||
成功 -> 返回持有锁的文件句柄(调用方须保持其存活,直到进程退出/主动释放);
|
||||
抢不到锁(已有其它 worker 持有)或打开失败 -> 返回 None。
|
||||
进程异常退出时 OS 会自动释放该锁。
|
||||
"""
|
||||
try:
|
||||
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
handle = open(lock_path, "a+")
|
||||
except OSError:
|
||||
logger.exception("Curator scheduler: 打开锁文件失败")
|
||||
return None
|
||||
try:
|
||||
if sys.platform == "win32":
|
||||
import msvcrt
|
||||
|
||||
handle.seek(0)
|
||||
msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1)
|
||||
else:
|
||||
import fcntl
|
||||
|
||||
fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except OSError:
|
||||
handle.close()
|
||||
return None
|
||||
return handle
|
||||
|
||||
|
||||
def _release_singleton_lock(handle) -> None:
|
||||
"""释放并关闭 _acquire_singleton_lock 拿到的锁文件句柄。"""
|
||||
try:
|
||||
if sys.platform == "win32":
|
||||
import msvcrt
|
||||
|
||||
handle.seek(0)
|
||||
msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
|
||||
else:
|
||||
import fcntl
|
||||
|
||||
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
|
||||
except OSError:
|
||||
pass
|
||||
finally:
|
||||
try:
|
||||
handle.close()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
async def _curator_scheduler_loop() -> None:
|
||||
"""常驻后台任务:每隔 _CURATOR_CHECK_INTERVAL_SECONDS 检查一次需不需要进化。
|
||||
|
||||
多 worker 部署下用文件锁做 leader 选举 —— 只有抢到锁的那个 worker 进程真正
|
||||
运行调度,其余 worker 直接退出,避免 curator 被重复触发、重复调用 LLM。
|
||||
被 cancel 时 CancelledError 会穿过 ``except Exception``(它是 BaseException
|
||||
子类,不会被捕获),协程随之干净退出并在 finally 里释放锁。
|
||||
"""
|
||||
from deerflow.config.runtime_paths import runtime_home
|
||||
from deerflow.skills.curator import maybe_run_curator
|
||||
|
||||
lock_handle = _acquire_singleton_lock(runtime_home() / ".curator_scheduler.lock")
|
||||
if lock_handle is None:
|
||||
logger.info("Curator scheduler: 另一个 worker 已持有调度锁,本 worker 跳过")
|
||||
return
|
||||
|
||||
logger.info("Curator scheduler: 取得调度锁,本 worker 负责 curator 调度")
|
||||
try:
|
||||
# 启动后延迟一会儿再首次检查,避开启动高峰
|
||||
await asyncio.sleep(_CURATOR_STARTUP_DELAY_SECONDS)
|
||||
while True:
|
||||
try:
|
||||
await maybe_run_curator()
|
||||
except Exception:
|
||||
logger.exception("Curator scheduler tick failed (non-fatal)")
|
||||
await asyncio.sleep(_CURATOR_CHECK_INTERVAL_SECONDS)
|
||||
finally:
|
||||
_release_singleton_lock(lock_handle)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
"""Application lifespan handler."""
|
||||
@@ -302,12 +390,12 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
except Exception:
|
||||
logger.exception("Skill ownership migration failed (non-fatal)")
|
||||
|
||||
# Kick off skill curator in background if the run interval has elapsed
|
||||
# 启动 skill curator 调度器:常驻后台任务,每小时检查一次需不需要进化
|
||||
# (per-user interval_hours 仍是真正决定要不要跑进化的闸)。
|
||||
try:
|
||||
from deerflow.skills.curator import maybe_run_curator
|
||||
asyncio.create_task(maybe_run_curator())
|
||||
app.state.curator_scheduler_task = asyncio.create_task(_curator_scheduler_loop())
|
||||
except Exception:
|
||||
logger.exception("Skill curator startup task failed (non-fatal)")
|
||||
logger.exception("Skill curator scheduler failed to start (non-fatal)")
|
||||
|
||||
# Start IM channel service if any channels are configured
|
||||
try:
|
||||
@@ -335,6 +423,17 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
|
||||
yield
|
||||
|
||||
# 停止 curator 调度器
|
||||
curator_task = getattr(app.state, "curator_scheduler_task", None)
|
||||
if curator_task is not None:
|
||||
curator_task.cancel()
|
||||
try:
|
||||
await curator_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception:
|
||||
logger.exception("Curator scheduler shutdown error")
|
||||
|
||||
try:
|
||||
scheduler_service = getattr(app.state, "scheduled_task_service", None)
|
||||
if scheduler_service is not None:
|
||||
|
||||
@@ -3,9 +3,50 @@ from pydantic import BaseModel, Field
|
||||
|
||||
from app.gateway.deps import get_config
|
||||
from deerflow.config.app_config import AppConfig
|
||||
from deerflow.config.model_config import ModelConfig
|
||||
|
||||
router = APIRouter(prefix="/api", tags=["models"])
|
||||
|
||||
_PROVIDER_KEYWORDS: list[tuple[str, str]] = [
|
||||
("langchain_openai", "openai"),
|
||||
("langchain_anthropic", "anthropic"),
|
||||
("langchain_google", "google"),
|
||||
("langchain_groq", "groq"),
|
||||
("langchain_mistralai", "mistral"),
|
||||
("langchain_cohere", "cohere"),
|
||||
("ollama", "ollama"),
|
||||
("vllm", "vllm"),
|
||||
]
|
||||
|
||||
_LOCAL_PROVIDERS = {"ollama", "vllm"}
|
||||
|
||||
|
||||
def _derive_provider(use: str) -> str | None:
|
||||
use_lower = use.lower()
|
||||
for keyword, provider in _PROVIDER_KEYWORDS:
|
||||
if keyword in use_lower:
|
||||
return provider
|
||||
return None
|
||||
|
||||
|
||||
def _build_model_response(model: ModelConfig) -> "ModelResponse":
|
||||
extra = model.model_extra or {}
|
||||
provider = str(extra["provider"]) if "provider" in extra else _derive_provider(model.use)
|
||||
is_local = bool(extra["is_local"]) if "is_local" in extra else (provider in _LOCAL_PROVIDERS)
|
||||
supports_tool_calling = bool(extra["supports_tool_calling"]) if "supports_tool_calling" in extra else True
|
||||
return ModelResponse(
|
||||
name=model.name,
|
||||
model=model.model,
|
||||
display_name=model.display_name,
|
||||
description=model.description,
|
||||
supports_thinking=model.supports_thinking,
|
||||
supports_reasoning_effort=model.supports_reasoning_effort,
|
||||
supports_vision=model.supports_vision,
|
||||
supports_tool_calling=supports_tool_calling,
|
||||
is_local=is_local,
|
||||
provider=provider,
|
||||
)
|
||||
|
||||
|
||||
class ModelResponse(BaseModel):
|
||||
"""Response model for model information."""
|
||||
@@ -16,6 +57,10 @@ class ModelResponse(BaseModel):
|
||||
description: str | None = Field(None, description="Model description")
|
||||
supports_thinking: bool = Field(default=False, description="Whether model supports thinking mode")
|
||||
supports_reasoning_effort: bool = Field(default=False, description="Whether model supports reasoning effort")
|
||||
supports_vision: bool = Field(default=False, description="Whether model supports image inputs")
|
||||
supports_tool_calling: bool = Field(default=True, description="Whether model supports tool/function calling")
|
||||
is_local: bool = Field(default=False, description="Whether this is a locally-hosted model")
|
||||
provider: str | None = Field(None, description="Model provider name (openai, anthropic, ollama, etc.)")
|
||||
|
||||
|
||||
class TokenUsageResponse(BaseModel):
|
||||
@@ -73,17 +118,7 @@ async def list_models(config: AppConfig = Depends(get_config)) -> ModelsListResp
|
||||
}
|
||||
```
|
||||
"""
|
||||
models = [
|
||||
ModelResponse(
|
||||
name=model.name,
|
||||
model=model.model,
|
||||
display_name=model.display_name,
|
||||
description=model.description,
|
||||
supports_thinking=model.supports_thinking,
|
||||
supports_reasoning_effort=model.supports_reasoning_effort,
|
||||
)
|
||||
for model in config.models
|
||||
]
|
||||
models = [_build_model_response(model) for model in config.models]
|
||||
return ModelsListResponse(
|
||||
models=models,
|
||||
token_usage=TokenUsageResponse(enabled=config.token_usage.enabled),
|
||||
@@ -122,11 +157,4 @@ async def get_model(model_name: str, config: AppConfig = Depends(get_config)) ->
|
||||
if model is None:
|
||||
raise HTTPException(status_code=404, detail=f"Model '{model_name}' not found")
|
||||
|
||||
return ModelResponse(
|
||||
name=model.name,
|
||||
model=model.model,
|
||||
display_name=model.display_name,
|
||||
description=model.description,
|
||||
supports_thinking=model.supports_thinking,
|
||||
supports_reasoning_effort=model.supports_reasoning_effort,
|
||||
)
|
||||
return _build_model_response(model)
|
||||
@@ -1045,12 +1045,21 @@ async def curator_status(request: Request) -> dict:
|
||||
@router.post("/curator/run", summary="Trigger Curator Run")
|
||||
async def run_curator(request: Request) -> dict:
|
||||
"""Manually trigger a curator review pass for the authenticated user
|
||||
(bypasses interval check, runs in a background thread)."""
|
||||
(bypasses the interval check, runs in a background thread).
|
||||
|
||||
Pre-checks the master switch / curator switch / run lock so the response
|
||||
accurately reflects whether a run actually started — instead of always
|
||||
reporting success. Returns ``{"triggered": bool, "reason": str | None}``.
|
||||
"""
|
||||
try:
|
||||
import threading
|
||||
from deerflow.skills.curator import _curator_thread_target
|
||||
|
||||
from deerflow.skills.curator import _curator_thread_target, manual_trigger_blocked_reason
|
||||
|
||||
user_id = _current_user_id(request)
|
||||
reason = manual_trigger_blocked_reason(user_id)
|
||||
if reason is not None:
|
||||
return {"triggered": False, "reason": reason}
|
||||
thread = threading.Thread(
|
||||
target=_curator_thread_target,
|
||||
args=(user_id,),
|
||||
@@ -1058,7 +1067,7 @@ async def run_curator(request: Request) -> dict:
|
||||
name=f"skill-curator-manual-{user_id}",
|
||||
)
|
||||
thread.start()
|
||||
return {"triggered": True, "message": "Curator run started in background"}
|
||||
return {"triggered": True, "reason": None}
|
||||
except Exception as e:
|
||||
logger.error("Failed to trigger curator: %s", e, exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=f"Failed to trigger curator: {str(e)}")
|
||||
@@ -1067,17 +1076,19 @@ async def run_curator(request: Request) -> dict:
|
||||
class CuratorSubConfig(BaseModel):
|
||||
"""Nested curator configuration mirroring CuratorConfig."""
|
||||
|
||||
enabled: bool = Field(default=True, description="Whether the curator runs automatically.")
|
||||
# 与 CuratorConfig.enabled 保持一致:默认关闭,需显式 opt-in。
|
||||
enabled: bool = Field(default=False, description="Whether the curator runs automatically.")
|
||||
interval_hours: int = Field(default=168, ge=1, description="Hours between curator runs.")
|
||||
stale_after_days: int = Field(default=30, ge=1, description="Days of inactivity before a skill is marked stale.")
|
||||
archive_after_days: int = Field(default=90, ge=1, description="Days of inactivity before a stale skill is archived.")
|
||||
review_timeout_seconds: int = Field(default=600, ge=1, description="Overall timeout (seconds) for the LLM review pass.")
|
||||
model_name: str | None = Field(default=None, description="Model name for the curator LLM review pass; null = primary model.")
|
||||
|
||||
|
||||
class CuratorConfigResponse(BaseModel):
|
||||
"""Response model exposing the effective skill_evolution config."""
|
||||
|
||||
enabled: bool
|
||||
moderation_model_name: str | None
|
||||
curator: CuratorSubConfig
|
||||
|
||||
|
||||
@@ -1089,7 +1100,6 @@ class CuratorConfigRequest(BaseModel):
|
||||
"""
|
||||
|
||||
enabled: bool = Field(..., description="Whether the agent can create and modify skills under skills/custom.")
|
||||
moderation_model_name: str | None = Field(default=None, description="Optional model name for skill security moderation.")
|
||||
curator: CuratorSubConfig
|
||||
|
||||
|
||||
@@ -1113,12 +1123,13 @@ def _build_curator_config_response(evo) -> CuratorConfigResponse:
|
||||
"""
|
||||
return CuratorConfigResponse(
|
||||
enabled=evo.enabled,
|
||||
moderation_model_name=evo.moderation_model_name,
|
||||
curator=CuratorSubConfig(
|
||||
enabled=evo.curator.enabled,
|
||||
interval_hours=evo.curator.interval_hours,
|
||||
stale_after_days=evo.curator.stale_after_days,
|
||||
archive_after_days=evo.curator.archive_after_days,
|
||||
review_timeout_seconds=evo.curator.review_timeout_seconds,
|
||||
model_name=evo.curator.model_name,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -98,6 +98,10 @@ def normalize_input(raw_input: dict[str, Any] | None) -> dict[str, Any]:
|
||||
|
||||
_DEFAULT_ASSISTANT_ID = "lead_agent"
|
||||
|
||||
# Built-in LangGraph graph IDs registered in langgraph.json.
|
||||
# These are NOT stored in the agents DB and must bypass the custom-agent access check.
|
||||
_BUILTIN_GRAPH_IDS: frozenset[str] = frozenset({"lead_agent", "canvas_agent"})
|
||||
|
||||
|
||||
def _prepend_prompt_prefix(graph_input: dict[str, Any], prefix: str) -> dict[str, Any]:
|
||||
"""Return ``graph_input`` with the prefix *tagged* onto its first HumanMessage.
|
||||
@@ -180,12 +184,15 @@ def merge_run_context_overrides(config: dict[str, Any], context: Mapping[str, An
|
||||
def resolve_agent_factory(assistant_id: str | None):
|
||||
"""Resolve the agent factory callable from config.
|
||||
|
||||
Custom agents are implemented as ``lead_agent`` + an ``agent_id``
|
||||
injected into ``configurable`` or ``context`` — see
|
||||
:func:`build_run_config`. All ``assistant_id`` values therefore map to the
|
||||
same factory; the routing happens inside ``make_lead_agent`` when it reads
|
||||
``cfg["agent_id"]``.
|
||||
Built-in graphs (``lead_agent``, ``canvas_agent``) are dispatched directly
|
||||
to their own factory. Custom agents share the ``lead_agent`` factory with
|
||||
``agent_id`` injected into configurable/context by :func:`build_run_config`.
|
||||
"""
|
||||
if assistant_id == "canvas_agent":
|
||||
from deerflow.agents.canvas_agent.agent import make_canvas_agent
|
||||
|
||||
return make_canvas_agent
|
||||
|
||||
from deerflow.agents.lead_agent.agent import make_lead_agent
|
||||
|
||||
return make_lead_agent
|
||||
@@ -244,8 +251,9 @@ def build_run_config(
|
||||
config["configurable"] = {"thread_id": thread_id}
|
||||
|
||||
# Inject custom agent id when the caller specified a non-default assistant.
|
||||
# Honour an explicit agent_id in the active runtime options container.
|
||||
if assistant_id and assistant_id != _DEFAULT_ASSISTANT_ID:
|
||||
# Built-in graphs (lead_agent, canvas_agent) are dispatched via their own
|
||||
# factory and must NOT receive an agent_id override.
|
||||
if assistant_id and assistant_id not in _BUILTIN_GRAPH_IDS:
|
||||
normalized = assistant_id.strip()
|
||||
if not normalized or not re.fullmatch(r"[A-Za-z0-9_-]+", normalized):
|
||||
raise ValueError(f"Invalid assistant_id {assistant_id!r}: must contain only letters, digits, hyphens, and underscores.")
|
||||
@@ -265,7 +273,7 @@ def build_run_config(
|
||||
def _collect_requested_agent_ids(body: Any) -> set[str]:
|
||||
"""Collect agent ids from assistant_id/config/context for access checks."""
|
||||
agent_ids: set[str] = set()
|
||||
if body.assistant_id and body.assistant_id != _DEFAULT_ASSISTANT_ID:
|
||||
if body.assistant_id and body.assistant_id not in _BUILTIN_GRAPH_IDS:
|
||||
agent_ids.add(str(body.assistant_id))
|
||||
|
||||
def collect(container: Mapping[str, Any] | None) -> None:
|
||||
|
||||
@@ -6,7 +6,8 @@
|
||||
],
|
||||
"env": ".env",
|
||||
"graphs": {
|
||||
"lead_agent": "deerflow.agents:make_lead_agent"
|
||||
"lead_agent": "deerflow.agents:make_lead_agent",
|
||||
"canvas_agent": "deerflow.agents.canvas_agent:make_canvas_agent"
|
||||
},
|
||||
"auth": {
|
||||
"path": "./app/gateway/langgraph_auth.py:auth"
|
||||
|
||||
+3
@@ -0,0 +1,3 @@
|
||||
from deerflow.agents.canvas_agent.agent import make_canvas_agent
|
||||
|
||||
__all__ = ["make_canvas_agent"]
|
||||
+68
@@ -0,0 +1,68 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from langchain.agents import create_agent
|
||||
from langchain_core.runnables import RunnableConfig
|
||||
|
||||
from deerflow.agents.canvas_agent.prompt import CANVAS_SYSTEM_PROMPT
|
||||
from deerflow.agents.canvas_agent.tools import update_artifact_tool
|
||||
from deerflow.agents.middlewares.clarification_middleware import ClarificationMiddleware
|
||||
from deerflow.agents.middlewares.dangling_tool_call_middleware import DanglingToolCallMiddleware
|
||||
from deerflow.agents.middlewares.loop_detection_middleware import LoopDetectionMiddleware
|
||||
from deerflow.agents.middlewares.title_middleware import TitleMiddleware
|
||||
from deerflow.agents.middlewares.tool_error_handling_middleware import ToolErrorHandlingMiddleware
|
||||
from deerflow.agents.thread_state import ThreadState
|
||||
from deerflow.config.app_config import get_app_config
|
||||
from deerflow.models import create_chat_model
|
||||
from deerflow.tools.builtins import ask_clarification_tool
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_runtime_config(config: RunnableConfig) -> dict:
|
||||
cfg = dict(config.get("configurable", {}) or {})
|
||||
context = config.get("context", {}) or {}
|
||||
if isinstance(context, dict):
|
||||
cfg.update(context)
|
||||
return cfg
|
||||
|
||||
|
||||
def make_canvas_agent(config: RunnableConfig):
|
||||
"""LangGraph graph factory for canvas mode; signature compatible with LangGraph Server."""
|
||||
app_config = get_app_config()
|
||||
cfg = _get_runtime_config(config)
|
||||
|
||||
requested_model = cfg.get("model_name") or cfg.get("model")
|
||||
if requested_model and app_config.get_model_config(requested_model):
|
||||
model_name = requested_model
|
||||
else:
|
||||
if requested_model:
|
||||
logger.warning("Canvas agent: model '%s' not found; falling back to default.", requested_model)
|
||||
model_name = app_config.models[0].name if app_config.models else None
|
||||
|
||||
if model_name is None:
|
||||
raise ValueError("No chat models configured. Please add at least one model in config.yaml.")
|
||||
|
||||
thinking_enabled = cfg.get("thinking_enabled", False)
|
||||
model_config = app_config.get_model_config(model_name)
|
||||
if thinking_enabled and model_config and not model_config.supports_thinking:
|
||||
thinking_enabled = False
|
||||
|
||||
logger.info("Create CanvasAgent -> model_name: %s, thinking_enabled: %s", model_name, thinking_enabled)
|
||||
|
||||
middlewares = [
|
||||
DanglingToolCallMiddleware(),
|
||||
ToolErrorHandlingMiddleware(),
|
||||
TitleMiddleware(app_config=app_config),
|
||||
LoopDetectionMiddleware(),
|
||||
ClarificationMiddleware(),
|
||||
]
|
||||
|
||||
return create_agent(
|
||||
model=create_chat_model(name=model_name, thinking_enabled=thinking_enabled, app_config=app_config),
|
||||
tools=[update_artifact_tool, ask_clarification_tool],
|
||||
middleware=middlewares,
|
||||
system_prompt=CANVAS_SYSTEM_PROMPT,
|
||||
state_schema=ThreadState,
|
||||
)
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
CANVAS_SYSTEM_PROMPT = """You are a Canvas AI assistant that helps users create and edit documents and code.
|
||||
|
||||
<canvas_behavior>
|
||||
You work with an **artifact** — a document or code file shown in the canvas panel on the right.
|
||||
ALWAYS call `update_artifact` to save your work. Never embed the artifact content in your chat response.
|
||||
|
||||
Rules:
|
||||
- On every request that creates or changes the artifact, call `update_artifact` ONCE with the COMPLETE updated content.
|
||||
- In your text reply, describe what you did in 1–2 sentences.
|
||||
- If the user is only asking a question (no edit needed), answer in text — skip the tool call.
|
||||
</canvas_behavior>
|
||||
|
||||
<artifact_context>
|
||||
The current artifact state is injected into your context. When editing, build on the existing content exactly.
|
||||
When creating from scratch, produce a complete, high-quality artifact in one shot.
|
||||
</artifact_context>
|
||||
|
||||
<selection_rewrite>
|
||||
When the user's message contains a <selection> block, you MUST:
|
||||
1. Identify the exact selection in the current artifact content.
|
||||
2. Rewrite ONLY that selection according to the user's instruction.
|
||||
3. Keep the rest of the artifact exactly unchanged.
|
||||
4. Call update_artifact with the complete artifact where the selection has been replaced.
|
||||
|
||||
Example: if the artifact is "A B C" and the selection is "B" with instruction "make it bold",
|
||||
produce "A **B** C" — not a new document.
|
||||
</selection_rewrite>
|
||||
|
||||
<quick_actions>
|
||||
When the user triggers a quick action, apply it to the full artifact unless specific highlighted text is provided:
|
||||
- expand: Add depth and detail while preserving structure
|
||||
- shorten: Condense to the essential points
|
||||
- formalize: Rewrite in professional, formal language
|
||||
- casualize: Rewrite in friendly, conversational language
|
||||
- add_comments: Add clear, helpful code comments
|
||||
- add_logs: Add appropriate logging statements
|
||||
- fix_bugs: Identify and fix all bugs in the code
|
||||
- optimize_code: Improve performance and readability
|
||||
</quick_actions>
|
||||
"""
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Annotated, Literal
|
||||
|
||||
from langchain.tools import InjectedToolCallId, ToolRuntime, tool
|
||||
from langchain_core.messages import ToolMessage
|
||||
from langgraph.types import Command
|
||||
from langgraph.typing import ContextT
|
||||
|
||||
from deerflow.agents.thread_state import CanvasArtifact, CanvasArtifactCode, CanvasArtifactText, ThreadState
|
||||
|
||||
|
||||
@tool("update_artifact", parse_docstring=True)
|
||||
def update_artifact_tool(
|
||||
runtime: ToolRuntime[ContextT, ThreadState],
|
||||
tool_call_id: Annotated[str, InjectedToolCallId],
|
||||
type: Literal["text", "code"],
|
||||
title: str,
|
||||
content: str,
|
||||
language: str = "other",
|
||||
) -> Command:
|
||||
"""Create or update the canvas artifact. Always call this to persist your work.
|
||||
|
||||
Args:
|
||||
type: Artifact type — "text" for markdown documents, "code" for source code.
|
||||
title: Human-readable title of the artifact.
|
||||
content: Full artifact content: markdown text or complete source code.
|
||||
language: Programming language (for code type only, e.g. "python", "typescript"). Ignored for text.
|
||||
"""
|
||||
current: CanvasArtifact | None = (runtime.state or {}).get("artifact") if runtime.state else None
|
||||
|
||||
if current is not None:
|
||||
existing_contents = list(current.get("contents", []))
|
||||
new_index = current.get("currentIndex", len(existing_contents)) + 1
|
||||
else:
|
||||
existing_contents = []
|
||||
new_index = 1
|
||||
|
||||
if type == "text":
|
||||
new_entry: CanvasArtifactText | CanvasArtifactCode = CanvasArtifactText(
|
||||
index=new_index,
|
||||
type="text",
|
||||
title=title,
|
||||
fullMarkdown=content,
|
||||
)
|
||||
else:
|
||||
new_entry = CanvasArtifactCode(
|
||||
index=new_index,
|
||||
type="code",
|
||||
title=title,
|
||||
language=language,
|
||||
code=content,
|
||||
)
|
||||
|
||||
updated: CanvasArtifact = {
|
||||
"currentIndex": new_index,
|
||||
"contents": existing_contents + [new_entry],
|
||||
}
|
||||
|
||||
return Command(
|
||||
update={
|
||||
"artifact": updated,
|
||||
"messages": [ToolMessage("Artifact updated.", tool_call_id=tool_call_id)],
|
||||
}
|
||||
)
|
||||
+66
-49
@@ -69,8 +69,25 @@ _enabled_skills_refresh_version = 0
|
||||
_enabled_skills_refresh_event = threading.Event()
|
||||
|
||||
|
||||
def _drop_archived_skills(skills: list[Skill]) -> list[Skill]:
|
||||
"""从注入提示词的技能列表里剔除已归档(usage state=archived)的技能。
|
||||
|
||||
技能归档是软状态:SKILL.md 仍留在 custom/<name>/、enabled 标志不变,所以
|
||||
load_skills(enabled_only=True) 仍会返回它。但归档技能不应再出现在 agent 的系统
|
||||
提示词里 —— 对 agent 而言它应等同于"不存在"。失败时不过滤(返回原列表)。
|
||||
"""
|
||||
try:
|
||||
from deerflow.skills.usage import STATE_ARCHIVED, all_entries
|
||||
|
||||
entries = all_entries()
|
||||
return [s for s in skills if entries.get(s.name, {}).get("state") != STATE_ARCHIVED]
|
||||
except Exception:
|
||||
logger.debug("Failed to filter archived skills for prompt injection", exc_info=True)
|
||||
return skills
|
||||
|
||||
|
||||
def _load_enabled_skills_sync() -> list[Skill]:
|
||||
return list(get_or_new_skill_storage().load_skills(enabled_only=True))
|
||||
return _drop_archived_skills(list(get_or_new_skill_storage().load_skills(enabled_only=True)))
|
||||
|
||||
|
||||
def _start_enabled_skills_refresh_thread() -> None:
|
||||
@@ -171,7 +188,7 @@ def _get_enabled_skills_for_config(app_config: AppConfig | None = None) -> list[
|
||||
"""
|
||||
if app_config is None:
|
||||
return _get_enabled_skills()
|
||||
return list(get_or_new_skill_storage(app_config=app_config).load_skills(enabled_only=True))
|
||||
return _drop_archived_skills(list(get_or_new_skill_storage(app_config=app_config).load_skills(enabled_only=True)))
|
||||
|
||||
|
||||
def _skill_mutability_label(category: SkillCategory | str) -> str:
|
||||
@@ -479,11 +496,11 @@ You: "Deploying to staging..." [proceed]
|
||||
|
||||
{skills_section}
|
||||
|
||||
{deferred_tools_section}
|
||||
|
||||
{writing_mode_section}
|
||||
|
||||
{subagent_section}
|
||||
{deferred_tools_section}
|
||||
|
||||
{writing_mode_section}
|
||||
|
||||
{subagent_section}
|
||||
|
||||
<working_directory existed="true">
|
||||
- User uploads: `/mnt/user-data/uploads` - Files uploaded by the user (automatically listed in context)
|
||||
@@ -796,17 +813,17 @@ def _build_custom_mounts_section(*, app_config: AppConfig | None = None) -> str:
|
||||
return f"\n**Custom Mounted Directories:**\n{mounts_list}\n- If the user needs files outside `/mnt/user-data`, use these absolute container paths directly when they match the requested directory"
|
||||
|
||||
|
||||
def apply_prompt_template(
|
||||
subagent_enabled: bool = False,
|
||||
max_concurrent_subagents: int = 3,
|
||||
*,
|
||||
agent_id: str | None = None,
|
||||
agent_name: str | None = None,
|
||||
writing_mode: bool = False,
|
||||
writing_artifact_path: str | None = None,
|
||||
available_skills: set[str] | None = None,
|
||||
app_config: AppConfig | None = None,
|
||||
) -> str:
|
||||
def apply_prompt_template(
|
||||
subagent_enabled: bool = False,
|
||||
max_concurrent_subagents: int = 3,
|
||||
*,
|
||||
agent_id: str | None = None,
|
||||
agent_name: str | None = None,
|
||||
writing_mode: bool = False,
|
||||
writing_artifact_path: str | None = None,
|
||||
available_skills: set[str] | None = None,
|
||||
app_config: AppConfig | None = None,
|
||||
) -> str:
|
||||
storage_agent_id = agent_id if agent_id is not None else agent_name
|
||||
|
||||
# Get memory context
|
||||
@@ -838,33 +855,33 @@ def apply_prompt_template(
|
||||
skills_section = get_skills_prompt_section(available_skills, app_config=app_config)
|
||||
|
||||
# Get deferred tools section (tool_search)
|
||||
deferred_tools_section = get_deferred_tools_prompt_section(app_config=app_config)
|
||||
current_writing_artifact = (
|
||||
f"\n**Current Markdown Artifact**\n- Continue editing this existing Markdown file unless the user explicitly asks for a new document: `{writing_artifact_path}`\n"
|
||||
if writing_artifact_path
|
||||
else ""
|
||||
)
|
||||
writing_mode_section = (
|
||||
f"""
|
||||
<writing_mode>
|
||||
Writing mode is enabled for this run.
|
||||
|
||||
**Mandatory Output Contract**
|
||||
- Treat the user's request as a writing/report-generation task.
|
||||
- Your final deliverable MUST be a Markdown file, not only an inline chat answer.
|
||||
- If a current Markdown artifact is provided below, you MUST revise that existing file rather than creating a new file, unless the user explicitly asks for a separate/new document.
|
||||
- When revising an existing Markdown artifact, read the file first if needed, then overwrite the same path with the complete updated Markdown content.
|
||||
- Write the deliverable to `/mnt/user-data/outputs/` with a clear `.md` filename, for example `/mnt/user-data/outputs/report.md`.
|
||||
- Use `write_file` or an equivalent file-writing tool to save the complete Markdown content.
|
||||
- After writing the file, call `present_files` with the Markdown file path so the user can open it as an artifact/canvas.
|
||||
- The final chat response should be brief and point to the generated Markdown artifact. Do not duplicate the full report inline unless the user explicitly asks.
|
||||
- If you revise an existing Markdown artifact, update the Markdown file and present it again.
|
||||
{current_writing_artifact}
|
||||
</writing_mode>
|
||||
"""
|
||||
if writing_mode
|
||||
else ""
|
||||
)
|
||||
deferred_tools_section = get_deferred_tools_prompt_section(app_config=app_config)
|
||||
current_writing_artifact = (
|
||||
f"\n**Current Markdown Artifact**\n- Continue editing this existing Markdown file unless the user explicitly asks for a new document: `{writing_artifact_path}`\n"
|
||||
if writing_artifact_path
|
||||
else ""
|
||||
)
|
||||
writing_mode_section = (
|
||||
f"""
|
||||
<writing_mode>
|
||||
Writing mode is enabled for this run.
|
||||
|
||||
**Mandatory Output Contract**
|
||||
- Treat the user's request as a writing/report-generation task.
|
||||
- Your final deliverable MUST be a Markdown file, not only an inline chat answer.
|
||||
- If a current Markdown artifact is provided below, you MUST revise that existing file rather than creating a new file, unless the user explicitly asks for a separate/new document.
|
||||
- When revising an existing Markdown artifact, read the file first if needed, then overwrite the same path with the complete updated Markdown content.
|
||||
- Write the deliverable to `/mnt/user-data/outputs/` with a clear `.md` filename, for example `/mnt/user-data/outputs/report.md`.
|
||||
- Use `write_file` or an equivalent file-writing tool to save the complete Markdown content.
|
||||
- After writing the file, call `present_files` with the Markdown file path so the user can open it as an artifact/canvas.
|
||||
- The final chat response should be brief and point to the generated Markdown artifact. Do not duplicate the full report inline unless the user explicitly asks.
|
||||
- If you revise an existing Markdown artifact, update the Markdown file and present it again.
|
||||
{current_writing_artifact}
|
||||
</writing_mode>
|
||||
"""
|
||||
if writing_mode
|
||||
else ""
|
||||
)
|
||||
|
||||
# Build ACP agent section only if ACP agents are configured
|
||||
acp_section = _build_acp_section(app_config=app_config)
|
||||
@@ -878,10 +895,10 @@ Writing mode is enabled for this run.
|
||||
prompt = SYSTEM_PROMPT_TEMPLATE.format(
|
||||
agent_name=agent_name or storage_agent_id or "CM 助手",
|
||||
soul=get_agent_soul(storage_agent_id),
|
||||
skills_section=skills_section,
|
||||
deferred_tools_section=deferred_tools_section,
|
||||
writing_mode_section=writing_mode_section,
|
||||
memory_context=memory_context,
|
||||
skills_section=skills_section,
|
||||
deferred_tools_section=deferred_tools_section,
|
||||
writing_mode_section=writing_mode_section,
|
||||
memory_context=memory_context,
|
||||
network_policy_section=network_policy_section,
|
||||
subagent_section=subagent_section,
|
||||
subagent_reminder=subagent_reminder,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from typing import Annotated, NotRequired, TypedDict
|
||||
from typing import Annotated, Literal, NotRequired, Optional, TypedDict
|
||||
|
||||
from langchain.agents import AgentState
|
||||
|
||||
@@ -18,6 +18,26 @@ class ViewedImageData(TypedDict):
|
||||
mime_type: str
|
||||
|
||||
|
||||
class CanvasArtifactText(TypedDict):
|
||||
index: int
|
||||
type: Literal["text"]
|
||||
title: str
|
||||
fullMarkdown: str
|
||||
|
||||
|
||||
class CanvasArtifactCode(TypedDict):
|
||||
index: int
|
||||
type: Literal["code"]
|
||||
title: str
|
||||
language: str
|
||||
code: str
|
||||
|
||||
|
||||
class CanvasArtifact(TypedDict):
|
||||
currentIndex: int
|
||||
contents: list[CanvasArtifactText | CanvasArtifactCode]
|
||||
|
||||
|
||||
def merge_artifacts(existing: list[str] | None, new: list[str] | None) -> list[str]:
|
||||
"""Reducer for artifacts list - merges and deduplicates artifacts."""
|
||||
if existing is None:
|
||||
@@ -53,3 +73,11 @@ class ThreadState(AgentState):
|
||||
todos: NotRequired[list | None]
|
||||
uploaded_files: NotRequired[list[dict] | None]
|
||||
viewed_images: Annotated[dict[str, ViewedImageData], merge_viewed_images] # image_path -> {base64, mime_type}
|
||||
# Canvas fields (coexist with artifacts: list[str] which tracks file outputs)
|
||||
artifact: NotRequired[Optional[CanvasArtifact]]
|
||||
highlighted_text: NotRequired[Optional[str]]
|
||||
highlighted_code: NotRequired[Optional[str]]
|
||||
canvas_mode: NotRequired[Optional[Literal["text", "code"]]]
|
||||
quick_action_id: NotRequired[Optional[str]]
|
||||
web_search_enabled: NotRequired[bool]
|
||||
web_search_results: NotRequired[Optional[list[dict]]]
|
||||
+8
-5
@@ -4,10 +4,17 @@ from pydantic import BaseModel, Field
|
||||
class CuratorConfig(BaseModel):
|
||||
"""Configuration for the background skill curator."""
|
||||
|
||||
enabled: bool = Field(default=True, description="Whether the curator runs automatically.")
|
||||
# 默认关闭:curator 仅在用户显式开启时才运行(配合 SkillEvolutionConfig.enabled
|
||||
# 总开关,二者均为安全默认 False —— 未显式 opt-in 则整套技能进化不动作)。
|
||||
enabled: bool = Field(default=False, description="Whether the curator runs automatically.")
|
||||
interval_hours: int = Field(default=168, description="Hours between curator runs (default: 7 days).")
|
||||
stale_after_days: int = Field(default=30, description="Days of inactivity before a skill is marked stale.")
|
||||
archive_after_days: int = Field(default=90, description="Days of inactivity before a stale skill is archived.")
|
||||
# LLM 评审整体超时:agent.invoke 内部可能多轮调 LLM,模型层的单次 request_timeout
|
||||
# 兜不住整体;超过此秒数则中止本次评审,curator 正常收尾并释放运行锁。
|
||||
review_timeout_seconds: int = Field(default=600, description="Overall timeout (seconds) for the LLM review pass.")
|
||||
# curator LLM 评审使用的模型名(对应 config.yaml models[].name);为空则用主模型。
|
||||
model_name: str | None = Field(default=None, description="Model name for the curator LLM review pass. Defaults to the primary chat model.")
|
||||
|
||||
|
||||
class SkillEvolutionConfig(BaseModel):
|
||||
@@ -17,10 +24,6 @@ class SkillEvolutionConfig(BaseModel):
|
||||
default=False,
|
||||
description="Whether the agent can create and modify skills under skills/custom.",
|
||||
)
|
||||
moderation_model_name: str | None = Field(
|
||||
default=None,
|
||||
description="Optional model name for skill security moderation. Defaults to the primary chat model.",
|
||||
)
|
||||
curator: CuratorConfig = Field(
|
||||
default_factory=CuratorConfig,
|
||||
description="Configuration for the background skill curator.",
|
||||
|
||||
@@ -67,3 +67,13 @@ class SkillStore(ABC):
|
||||
async def count_owned(self, user_id: str) -> int:
|
||||
"""Return the number of skills owned by ``user_id`` (owner_user_id IS NOT NULL)."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_published_names(self) -> set[str]:
|
||||
"""Return the set of skill names with ``published = True`` (any owner).
|
||||
|
||||
Batch alternative to calling ``get_any`` per skill — the curator uses
|
||||
this to filter out published skills in a single query instead of one
|
||||
DB round-trip per candidate skill.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
+3
@@ -133,3 +133,6 @@ class MemorySkillStore(SkillStore):
|
||||
|
||||
async def count_owned(self, user_id: str) -> int:
|
||||
return sum(1 for s in self._skills.values() if s.get("owner_user_id") == user_id)
|
||||
|
||||
async def list_published_names(self) -> set[str]:
|
||||
return {name for name, record in self._skills.items() if record.get("published")}
|
||||
@@ -148,3 +148,9 @@ class SkillRepository(SkillStore):
|
||||
stmt = select(func.count()).select_from(SkillRow).where(SkillRow.owner_user_id == user_id)
|
||||
async with self._sf() as session:
|
||||
return await session.scalar(stmt) or 0
|
||||
|
||||
async def list_published_names(self) -> set[str]:
|
||||
stmt = select(SkillRow.name).where(SkillRow.published.is_(True))
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return {row[0] for row in result.all()}
|
||||
@@ -10,6 +10,7 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import concurrent.futures
|
||||
import contextvars
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
@@ -25,6 +26,10 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
_CURATOR_STATE_FILENAME = ".curator_state"
|
||||
|
||||
# 周期调度一轮里同时执行的 curator 数量上限。到期用户多时(尤其首次部署所有用户一起
|
||||
# 到期)用线程池排队消费,避免一次性起 N 个线程齐调 LLM 打爆 provider 速率限制。
|
||||
_CURATOR_MAX_CONCURRENCY = 5
|
||||
|
||||
_T = TypeVar("_T")
|
||||
|
||||
|
||||
@@ -119,6 +124,81 @@ def _write_curator_state(state: dict, user_id: str | None) -> None:
|
||||
logger.debug("Failed to write curator state", exc_info=True)
|
||||
|
||||
|
||||
def _parse_state_time(value: str | None) -> datetime | None:
|
||||
"""解析 ``.curator_state`` 里的 ISO 时间字符串,返回 aware datetime(UTC)。
|
||||
|
||||
naive(无时区)字符串按 UTC 补全 —— 与 ``apply_automatic_transitions`` 的处理
|
||||
一致,避免 naive 时间串与 ``datetime.now(UTC)`` 直接比较时抛异常。无法解析时
|
||||
返回 None,由调用方决定降级行为。
|
||||
"""
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
dt = datetime.fromisoformat(value)
|
||||
except Exception:
|
||||
return None
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=UTC)
|
||||
return dt
|
||||
|
||||
|
||||
# Curator 运行锁:防止同一用户的 curator 并发执行(定时器 vs 手动触发、连点手动触发)。
|
||||
# 锁状态写进 .curator_state 的 running/started_at 字段,跨 worker 可见(手动请求可能
|
||||
# 落到非调度 worker)。started_at 超过下面的阈值视为进程残留,允许接管以免永久死锁。
|
||||
_CURATOR_STALE_LOCK_HOURS = 6
|
||||
# 上一轮"尝试"过但未成功(last_run_at 始终为空)时的重试间隔 —— 比正常 interval_hours
|
||||
# 短得多,但也不至于每个调度 tick(1h)都重试、每次都烧一份 LLM 配额。
|
||||
_CURATOR_RETRY_HOURS = 6
|
||||
# 进程内串行化运行锁的"读-改-写",保证同进程内的抢锁是原子的(跨进程的极小竞态窗口
|
||||
# 可接受 —— curator 触发频率极低)。
|
||||
_run_lock_guard = threading.Lock()
|
||||
|
||||
|
||||
def _run_lock_active(state: dict, now: datetime) -> bool:
|
||||
"""判断 ``.curator_state`` 里的运行锁当前是否有效(running 且未过期)。
|
||||
|
||||
``started_at`` 缺失或不可解析、或距今已超过 ``_CURATOR_STALE_LOCK_HOURS`` 时,
|
||||
视为进程残留留下的死锁,返回 False 以便后续触发能接管。
|
||||
"""
|
||||
if not state.get("running"):
|
||||
return False
|
||||
started_dt = _parse_state_time(state.get("started_at"))
|
||||
if started_dt is None:
|
||||
return False
|
||||
return now < started_dt + timedelta(hours=_CURATOR_STALE_LOCK_HOURS)
|
||||
|
||||
|
||||
def _try_acquire_run_lock(user_id: str, now: datetime) -> bool:
|
||||
"""尝试为 ``user_id`` 抢占 curator 运行锁。
|
||||
|
||||
抢到锁(已把 running=True / started_at 写回)返回 True;已有未过期的运行时返回
|
||||
False。读-改-写在 ``_run_lock_guard`` 下进行,保证同进程内原子。
|
||||
"""
|
||||
with _run_lock_guard:
|
||||
state = _read_curator_state(user_id)
|
||||
if _run_lock_active(state, now):
|
||||
return False
|
||||
state["running"] = True
|
||||
state["started_at"] = now.isoformat()
|
||||
_write_curator_state(state, user_id)
|
||||
return True
|
||||
|
||||
|
||||
def _release_run_lock(user_id: str) -> None:
|
||||
"""释放 curator 运行锁并记录本次尝试时间。无论运行成功或异常都应调用。
|
||||
|
||||
除了把 running 置回 False,还写入 ``last_attempt_at`` —— 用于失败退避:首次运行
|
||||
若一直崩溃,``last_run_at`` 永远为空,``should_run_now`` 改用
|
||||
``last_attempt_at + _CURATOR_RETRY_HOURS`` 判定,避免每个 tick 都重试。
|
||||
成功跑完后 ``last_run_at`` 已被写入,``should_run_now`` 不再看 ``last_attempt_at``。
|
||||
"""
|
||||
with _run_lock_guard:
|
||||
state = _read_curator_state(user_id)
|
||||
state["running"] = False
|
||||
state["last_attempt_at"] = datetime.now(UTC).isoformat()
|
||||
_write_curator_state(state, user_id)
|
||||
|
||||
|
||||
def _get_curator_config(user_id: str | None = None):
|
||||
"""Return the curator config object for ``user_id``.
|
||||
|
||||
@@ -135,6 +215,21 @@ def _get_curator_config(user_id: str | None = None):
|
||||
return None
|
||||
|
||||
|
||||
def _skill_evolution_enabled(user_id: str | None = None) -> bool:
|
||||
"""返回技能进化总开关 ``skill_evolution.enabled``(已合并 per-user 覆盖)。
|
||||
|
||||
这是整套技能进化功能的总闸:关闭时 curator 完全不动作。任何读取异常都按
|
||||
"关闭"处理(fail-safe)—— 宁可不跑,也不在配置不可读时偷偷运行。
|
||||
"""
|
||||
try:
|
||||
from deerflow.config.skill_evolution_runtime import load_skill_evolution_config_for_user
|
||||
|
||||
skill_evo = load_skill_evolution_config_for_user(user_id)
|
||||
return bool(getattr(skill_evo, "enabled", False))
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _list_user_owned_skill_names(user_id: str) -> set[str] | None:
|
||||
"""Return skill names owned by ``user_id`` (used to filter transitions).
|
||||
|
||||
@@ -231,12 +326,39 @@ def _is_skill_published(skill_name: str) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def _list_published_skill_names() -> set[str]:
|
||||
"""一次性返回所有 ``published=True`` 的技能名集合。
|
||||
|
||||
取代"每个技能调一次 ``_is_skill_published``"的逐条查库 —— curator 一轮里会遍历
|
||||
所有候选技能,逐条查会产生 ≈2N 次独立 event loop + DB 往返。出错时返回空集
|
||||
(fail-open:与 ``_is_skill_published`` 出错返回 False 语义一致,即 DB 不可用时
|
||||
所有技能都视为未发布,回退到 curator 的 legacy 行为)。
|
||||
"""
|
||||
try:
|
||||
from deerflow.persistence.engine import get_session_factory
|
||||
from deerflow.persistence.skills import make_skill_store
|
||||
|
||||
factory = get_session_factory()
|
||||
if factory is None:
|
||||
return set()
|
||||
store = make_skill_store(factory)
|
||||
|
||||
async def _run() -> set[str]:
|
||||
return await store.list_published_names()
|
||||
|
||||
return _run_async_blocking(_run())
|
||||
except Exception:
|
||||
logger.debug("_list_published_skill_names failed", exc_info=True)
|
||||
return set()
|
||||
|
||||
|
||||
def _eligible_for_curation(
|
||||
name: str,
|
||||
entry: dict,
|
||||
*,
|
||||
owned_set: set[str] | None,
|
||||
referenced_set: set[str],
|
||||
published_set: set[str],
|
||||
) -> bool:
|
||||
"""Three-condition gate for whether curator may touch a skill.
|
||||
|
||||
@@ -244,11 +366,15 @@ def _eligible_for_curation(
|
||||
1. source == "agent" (caller is expected to have already filtered, but
|
||||
we double-check here for safety)
|
||||
2. NOT referenced by any agent's skills config
|
||||
3. NOT published (queried via SkillStore)
|
||||
3. NOT published
|
||||
|
||||
``owned_set`` is the set of skill names owned by the running user. When
|
||||
None, the legacy "process every agent-created skill" fallback applies
|
||||
for ownership only (the other two conditions still hold).
|
||||
|
||||
``published_set`` is the set of published skill names, pre-fetched once by
|
||||
the caller via ``_list_published_skill_names()`` — avoids a per-skill DB
|
||||
round-trip.
|
||||
"""
|
||||
if entry.get("source") != "agent":
|
||||
return False
|
||||
@@ -256,7 +382,7 @@ def _eligible_for_curation(
|
||||
return False
|
||||
if name in referenced_set:
|
||||
return False
|
||||
if _is_skill_published(name):
|
||||
if name in published_set:
|
||||
return False
|
||||
return True
|
||||
|
||||
@@ -264,21 +390,52 @@ def _eligible_for_curation(
|
||||
def should_run_now(user_id: str) -> bool:
|
||||
"""Return True if the curator interval has elapsed for ``user_id`` and it
|
||||
is not paused."""
|
||||
curator_cfg = _get_curator_config()
|
||||
if curator_cfg is None or not getattr(curator_cfg, "enabled", True):
|
||||
# 总开关:技能进化整体关闭时,curator 一律不跑。
|
||||
if not _skill_evolution_enabled(user_id):
|
||||
return False
|
||||
curator_cfg = _get_curator_config(user_id)
|
||||
if curator_cfg is None or not getattr(curator_cfg, "enabled", False):
|
||||
return False
|
||||
state = _read_curator_state(user_id)
|
||||
if state.get("paused"):
|
||||
return False
|
||||
# 运行锁:已有 curator 在跑(且锁未过期)时不再触发新的一轮。
|
||||
if _run_lock_active(state, datetime.now(UTC)):
|
||||
return False
|
||||
last_run = state.get("last_run_at")
|
||||
if not last_run:
|
||||
return True
|
||||
# 从未成功跑完过。若此前已尝试过(last_attempt_at 存在),按较短的重试间隔
|
||||
# 退避 —— 避免首次运行持续失败时每个 tick(1h)都重试、每次烧一份 LLM 配额。
|
||||
attempt_dt = _parse_state_time(state.get("last_attempt_at"))
|
||||
if attempt_dt is None:
|
||||
return True
|
||||
return datetime.now(UTC) >= attempt_dt + timedelta(hours=_CURATOR_RETRY_HOURS)
|
||||
interval_hours = getattr(curator_cfg, "interval_hours", 168)
|
||||
try:
|
||||
last_dt = datetime.fromisoformat(last_run)
|
||||
return datetime.now(UTC) >= last_dt + timedelta(hours=interval_hours)
|
||||
except Exception:
|
||||
last_dt = _parse_state_time(last_run)
|
||||
if last_dt is None:
|
||||
# 时间串损坏 —— 视同"从未跑过",允许重跑并重写正确时间戳,而不是像旧逻辑那样
|
||||
# except 返回 False 导致该用户永久不再触发。
|
||||
return True
|
||||
return datetime.now(UTC) >= last_dt + timedelta(hours=interval_hours)
|
||||
|
||||
|
||||
def manual_trigger_blocked_reason(user_id: str) -> str | None:
|
||||
"""手动触发 curator 的预检。返回 None 表示可以跑;否则返回阻塞原因代码:
|
||||
``evolution_disabled`` / ``curator_disabled`` / ``already_running``。
|
||||
|
||||
供 ``POST /api/curator/run`` 在派发后台线程**前**同步判定 —— 后台线程
|
||||
(``_curator_thread_target``)本身也会再查一遍,这里只是为了让前端拿到准确反馈,
|
||||
而不是无论开关状态如何都提示"已开始"。
|
||||
"""
|
||||
if not _skill_evolution_enabled(user_id):
|
||||
return "evolution_disabled"
|
||||
curator_cfg = _get_curator_config(user_id)
|
||||
if curator_cfg is None or not getattr(curator_cfg, "enabled", False):
|
||||
return "curator_disabled"
|
||||
state = _read_curator_state(user_id)
|
||||
if _run_lock_active(state, datetime.now(UTC)):
|
||||
return "already_running"
|
||||
return None
|
||||
|
||||
|
||||
def apply_automatic_transitions(user_id: str, now: datetime | None = None) -> dict:
|
||||
@@ -297,12 +454,13 @@ def apply_automatic_transitions(user_id: str, now: datetime | None = None) -> di
|
||||
if now is None:
|
||||
now = datetime.now(UTC)
|
||||
|
||||
curator_cfg = _get_curator_config()
|
||||
curator_cfg = _get_curator_config(user_id)
|
||||
stale_days = getattr(curator_cfg, "stale_after_days", 30) if curator_cfg else 30
|
||||
archive_days = getattr(curator_cfg, "archive_after_days", 90) if curator_cfg else 90
|
||||
|
||||
owned = _list_user_owned_skill_names(user_id)
|
||||
referenced = _collect_referenced_skill_names()
|
||||
published = _list_published_skill_names() # 一次性查出,避免逐技能查库
|
||||
raw_entries = agent_created_entries()
|
||||
|
||||
skipped_referenced = 0
|
||||
@@ -314,7 +472,7 @@ def apply_automatic_transitions(user_id: str, now: datetime | None = None) -> di
|
||||
if name in referenced:
|
||||
skipped_referenced += 1
|
||||
continue
|
||||
if _is_skill_published(name):
|
||||
if name in published:
|
||||
skipped_published += 1
|
||||
continue
|
||||
eligible[name] = entry
|
||||
@@ -377,9 +535,10 @@ def _render_candidate_list(user_id: str) -> str:
|
||||
|
||||
owned = _list_user_owned_skill_names(user_id)
|
||||
referenced = _collect_referenced_skill_names()
|
||||
published = _list_published_skill_names() # 一次性查出,避免逐技能查库
|
||||
entries: dict[str, dict] = {}
|
||||
for k, v in all_entries().items():
|
||||
if not _eligible_for_curation(k, v, owned_set=owned, referenced_set=referenced):
|
||||
if not _eligible_for_curation(k, v, owned_set=owned, referenced_set=referenced, published_set=published):
|
||||
continue
|
||||
if v.get("state") == STATE_ARCHIVED:
|
||||
continue
|
||||
@@ -590,8 +749,72 @@ def _make_skill_archive_tool(user_id: str):
|
||||
return _skill_archive
|
||||
|
||||
|
||||
def _run_llm_review(candidate_list: str, *, user_id: str) -> str:
|
||||
"""Run the curator consolidation pass as a React agent with skill tools bound."""
|
||||
def _make_curator_skill_manage_tool():
|
||||
"""curator 受限版 ``skill_manage``:在工具层屏蔽 delete 动作。
|
||||
|
||||
curator 做 umbrella 合并需要 create / patch / edit / write_file,但绝不能删整个
|
||||
技能 —— prompt 硬规则也写明"归档(skill_archive)是允许的最大操作"。通用
|
||||
skill_manage 的 delete 走 ``_assert_agent_may_delete``,**绕过了** skill_archive
|
||||
对"已发布 / 被 agent 引用"技能的拒绝保护。只靠 prompt 文字约束不可靠,这里在
|
||||
工具层直接拒掉 delete,curator 的 LLM 无论如何都删不掉技能。
|
||||
"""
|
||||
from langchain.tools import tool as lc_tool
|
||||
|
||||
from deerflow.tools.skill_manage_tool import _skill_manage_impl
|
||||
|
||||
@lc_tool("skill_manage")
|
||||
def _curator_skill_manage(
|
||||
action: str,
|
||||
name: str,
|
||||
content: str | None = None,
|
||||
path: str | None = None,
|
||||
find: str | None = None,
|
||||
replace: str | None = None,
|
||||
expected_count: int | None = None,
|
||||
) -> str:
|
||||
"""管理 skills/custom/ 下的自定义技能(curator 受限版)。
|
||||
|
||||
支持 action:create / patch / edit / write_file / remove_file / list / view。
|
||||
**不支持 delete** —— curator 不得删除技能;如需下架请改用 skill_archive。
|
||||
|
||||
Args:
|
||||
action: 操作类型(见上;delete 会被拒绝)。
|
||||
name: 连字符命名的技能名。
|
||||
content: create / edit / write_file 的新文件内容。
|
||||
path: write_file / remove_file 的支持文件路径。
|
||||
find: patch 要替换的原文本。
|
||||
replace: patch 的替换文本。
|
||||
expected_count: patch 的预期替换次数(可选)。
|
||||
"""
|
||||
if action == "delete":
|
||||
return (
|
||||
"Refused: curator 不允许删除技能。归档是允许的最大操作 —— "
|
||||
"请改用 skill_archive(name, reason) 软下架(文件保留,用户可在 Web UI 恢复)。"
|
||||
)
|
||||
# curator 不绑定到具体会话线程,runtime 传 None(_get_thread_id 兼容 None)。
|
||||
return _run_async_blocking(
|
||||
_skill_manage_impl(
|
||||
runtime=None,
|
||||
action=action,
|
||||
name=name,
|
||||
content=content,
|
||||
path=path,
|
||||
find=find,
|
||||
replace=replace,
|
||||
expected_count=expected_count,
|
||||
)
|
||||
)
|
||||
|
||||
return _curator_skill_manage
|
||||
|
||||
|
||||
def _run_llm_review(candidate_list: str, *, user_id: str) -> tuple[str, list[dict]]:
|
||||
"""Run the curator consolidation pass as a React agent with skill tools bound.
|
||||
|
||||
返回 ``(llm_text, tool_actions)``:
|
||||
- ``llm_text`` —— LLM 最终回复文字(它"自述"做了什么);
|
||||
- ``tool_actions`` —— 从消息流提取的**真实**工具调用记录(它"实际"做了什么)。
|
||||
"""
|
||||
from deerflow.skills import curator_log
|
||||
|
||||
try:
|
||||
@@ -601,10 +824,14 @@ def _run_llm_review(candidate_list: str, *, user_id: str) -> str:
|
||||
from deerflow.agents.middlewares.tool_error_handling_middleware import ToolErrorHandlingMiddleware
|
||||
from deerflow.models import create_chat_model
|
||||
from deerflow.tools.builtins.skill_tools import skill_list_tool, skill_view_tool
|
||||
from deerflow.tools.skill_manage_tool import skill_manage_tool
|
||||
|
||||
model = create_chat_model()
|
||||
tools = [skill_list_tool, skill_view_tool, skill_manage_tool, _make_skill_archive_tool(user_id)]
|
||||
curator_cfg = _get_curator_config(user_id)
|
||||
# curator 评审用的模型:per-user 可配 curator.model_name(对应 config.yaml
|
||||
# models[].name);为空则用主模型。
|
||||
model_name = getattr(curator_cfg, "model_name", None) if curator_cfg else None
|
||||
model = create_chat_model(name=model_name)
|
||||
# 用 curator 受限版 skill_manage(屏蔽 delete),不用通用版 —— 见 _make_curator_skill_manage_tool。
|
||||
tools = [skill_list_tool, skill_view_tool, _make_curator_skill_manage_tool(), _make_skill_archive_tool(user_id)]
|
||||
# Reuse the lead-agent error middleware: when the LLM mis-calls a tool
|
||||
# (wrong action / missing required args), the exception is converted
|
||||
# into a ToolMessage(status="error") so the LLM can read the error and
|
||||
@@ -613,30 +840,58 @@ def _run_llm_review(candidate_list: str, *, user_id: str) -> str:
|
||||
# middleware kwarg — use LangChain 1.x's ``create_agent`` which does.
|
||||
agent = create_agent(model=model, tools=tools, middleware=[ToolErrorHandlingMiddleware()])
|
||||
|
||||
curator_log.record("info", "LLM 评审:开始调用 React Agent", user_id=user_id)
|
||||
timeout_seconds = getattr(curator_cfg, "review_timeout_seconds", 600) if curator_cfg else 600
|
||||
curator_log.record(
|
||||
"info",
|
||||
f"LLM 评审:开始调用 React Agent(模型 {model_name or '主模型'},整体超时 {timeout_seconds}s)",
|
||||
user_id=user_id,
|
||||
)
|
||||
prompt = _CURATOR_REVIEW_PROMPT + f"\n\nSkill candidates for this pass:\n{candidate_list}"
|
||||
result = agent.invoke({"messages": [HumanMessage(content=prompt)]})
|
||||
|
||||
# 给整个 React Agent 评审套一个总超时:agent.invoke 内部可能多轮调 LLM,模型层的
|
||||
# 单次 request_timeout 兜不住整体。超时后本 curator 线程能正常收尾、释放运行锁,
|
||||
# 而不是无限期挂起。注意:Python 无法强杀线程 —— 超时后那个 agent.invoke 线程会
|
||||
# 成为孤儿(最终被模型层 request_timeout 或自然返回所终结),但不再拖住 curator。
|
||||
# copy_context():把 current_user 等 ContextVar 传进工作线程,否则工具调用会把
|
||||
# user_id 误解析为 DEFAULT_USER_ID。
|
||||
ctx = contextvars.copy_context()
|
||||
pool = concurrent.futures.ThreadPoolExecutor(max_workers=1, thread_name_prefix="curator-llm")
|
||||
future = pool.submit(ctx.run, agent.invoke, {"messages": [HumanMessage(content=prompt)]})
|
||||
try:
|
||||
result = future.result(timeout=timeout_seconds)
|
||||
except concurrent.futures.TimeoutError:
|
||||
curator_log.record(
|
||||
"warning",
|
||||
f"LLM 评审超时(超过 {timeout_seconds}s),已中止本次评审",
|
||||
user_id=user_id,
|
||||
)
|
||||
logger.warning("Curator[%s] LLM review timed out after %ss", user_id, timeout_seconds)
|
||||
return "", []
|
||||
finally:
|
||||
# wait=False:超时时不阻塞等待挂死的 invoke 线程。
|
||||
pool.shutdown(wait=False)
|
||||
|
||||
messages = result.get("messages", [])
|
||||
tool_actions = _extract_tool_actions(messages)
|
||||
for msg in reversed(messages):
|
||||
if not getattr(msg, "tool_calls", None):
|
||||
content = getattr(msg, "content", "")
|
||||
if isinstance(content, str) and content.strip():
|
||||
curator_log.record("info", "LLM 评审:Agent 已返回最终结论", user_id=user_id)
|
||||
return content
|
||||
return content, tool_actions
|
||||
if isinstance(content, list):
|
||||
text = " ".join(
|
||||
block.get("text", "") if isinstance(block, dict) else str(block)
|
||||
for block in content
|
||||
).strip()
|
||||
curator_log.record("info", "LLM 评审:Agent 已返回最终结论", user_id=user_id)
|
||||
return text
|
||||
return text, tool_actions
|
||||
curator_log.record("warning", "LLM 评审:本次未产出最终回复", user_id=user_id)
|
||||
return ""
|
||||
return "", tool_actions
|
||||
except Exception as e:
|
||||
logger.warning("Curator LLM review failed: %s", e)
|
||||
curator_log.record("warning", f"LLM 评审失败:{e}", user_id=user_id)
|
||||
return ""
|
||||
return "", []
|
||||
|
||||
|
||||
def _parse_structured_summary(text: str) -> dict:
|
||||
@@ -653,6 +908,44 @@ def _parse_structured_summary(text: str) -> dict:
|
||||
return {"consolidations": [], "prunings": []}
|
||||
|
||||
|
||||
def _extract_tool_actions(messages: list) -> list[dict]:
|
||||
"""从 agent 消息流里提取**真实发生的**工具调用 + 结果。
|
||||
|
||||
与 _parse_structured_summary(解析 LLM 自述的 YAML)互补:这里记录的是 curator
|
||||
实际调了哪些工具、入参是什么、成功还是失败 —— 用于审计"嘴上说的"与"实际做的"
|
||||
是否一致。每条形如 ``{tool, args, status, result}``。
|
||||
"""
|
||||
# 先按 tool_call_id 收集每次调用的工具名与入参(来自 AIMessage.tool_calls)
|
||||
calls: dict[str, dict] = {}
|
||||
for msg in messages:
|
||||
for tc in getattr(msg, "tool_calls", None) or []:
|
||||
tc_id = tc.get("id") if isinstance(tc, dict) else getattr(tc, "id", None)
|
||||
if not tc_id:
|
||||
continue
|
||||
calls[tc_id] = {
|
||||
"tool": tc.get("name") if isinstance(tc, dict) else getattr(tc, "name", None),
|
||||
"args": (tc.get("args") if isinstance(tc, dict) else getattr(tc, "args", None)) or {},
|
||||
}
|
||||
# 再按 ToolMessage(带 tool_call_id)收集真实结果
|
||||
actions: list[dict] = []
|
||||
for msg in messages:
|
||||
tc_id = getattr(msg, "tool_call_id", None)
|
||||
if not tc_id:
|
||||
continue
|
||||
call = calls.get(tc_id, {})
|
||||
raw = getattr(msg, "content", "")
|
||||
result = raw if isinstance(raw, str) else str(raw)
|
||||
actions.append(
|
||||
{
|
||||
"tool": call.get("tool") or getattr(msg, "name", None),
|
||||
"args": call.get("args", {}),
|
||||
"status": getattr(msg, "status", None) or "success",
|
||||
"result": result[:500],
|
||||
}
|
||||
)
|
||||
return actions
|
||||
|
||||
|
||||
def run_curator_review(user_id: str) -> dict:
|
||||
"""Main curator run for ``user_id``: transitions → LLM consolidation → state.
|
||||
|
||||
@@ -684,16 +977,23 @@ def run_curator_review(user_id: str) -> dict:
|
||||
"info", f"进入 LLM 评审的候选技能:共 {candidate_line_count} 个", user_id=user_id
|
||||
)
|
||||
|
||||
llm_text = _run_llm_review(candidate_list, user_id=user_id)
|
||||
suggestions = _parse_structured_summary(llm_text)
|
||||
curator_log.record(
|
||||
"info",
|
||||
(
|
||||
f"LLM 建议已解析:建议合并 {len(suggestions['consolidations'])} 项,"
|
||||
f"建议删减 {len(suggestions['prunings'])} 项"
|
||||
),
|
||||
user_id=user_id,
|
||||
)
|
||||
if candidate_line_count == 0:
|
||||
# 没有候选技能 —— 跳过 LLM 评审,省掉无意义的模型调用。
|
||||
# 年龄流转(上面的 apply_automatic_transitions)已完成,且不依赖 LLM。
|
||||
curator_log.record("info", "无候选技能,跳过 LLM 评审", user_id=user_id)
|
||||
suggestions = {"consolidations": [], "prunings": []}
|
||||
tool_actions: list[dict] = []
|
||||
else:
|
||||
llm_text, tool_actions = _run_llm_review(candidate_list, user_id=user_id)
|
||||
suggestions = _parse_structured_summary(llm_text)
|
||||
curator_log.record(
|
||||
"info",
|
||||
(
|
||||
f"LLM 建议已解析:建议合并 {len(suggestions['consolidations'])} 项,"
|
||||
f"建议删减 {len(suggestions['prunings'])} 项"
|
||||
),
|
||||
user_id=user_id,
|
||||
)
|
||||
# Surface per-item reasoning so operators can audit *why* each action
|
||||
# was taken, not just the count.
|
||||
for item in suggestions.get("consolidations", []) or []:
|
||||
@@ -718,15 +1018,26 @@ def run_curator_review(user_id: str) -> dict:
|
||||
user_id=user_id,
|
||||
)
|
||||
|
||||
# 上面是 LLM"自述"的建议;下面记录它"实际"调用了哪些工具,供对照审计。
|
||||
curator_log.record("info", f"LLM 实际调用工具 {len(tool_actions)} 次", user_id=user_id)
|
||||
for act in tool_actions:
|
||||
target = act.get("args", {}).get("name", "")
|
||||
curator_log.record(
|
||||
"info",
|
||||
f" 实际调用 [{act.get('tool')}]{f' [{target}]' if target else ''} → {act.get('status')}",
|
||||
user_id=user_id,
|
||||
)
|
||||
|
||||
state = _read_curator_state(user_id)
|
||||
state["last_run_at"] = now.isoformat()
|
||||
state["run_count"] = state.get("run_count", 0) + 1
|
||||
state["last_run_summary"] = (
|
||||
f"自动流转:标记陈旧 {transition_counters['marked_stale']} 个、"
|
||||
f"归档 {transition_counters['archived']} 个;"
|
||||
f"LLM:建议合并 {len(suggestions['consolidations'])} 项"
|
||||
f"LLM:建议合并 {len(suggestions['consolidations'])} 项、实际工具调用 {len(tool_actions)} 次"
|
||||
)
|
||||
state["last_consolidation_suggestions"] = suggestions
|
||||
state["last_consolidation_suggestions"] = suggestions # LLM 自述
|
||||
state["last_actual_actions"] = tool_actions # 真实工具调用记录
|
||||
_write_curator_state(state, user_id)
|
||||
|
||||
logger.info("Curator[%s]: run complete — %s", user_id, state["last_run_summary"])
|
||||
@@ -755,6 +1066,23 @@ def _curator_thread_target(user_id: str) -> None:
|
||||
from deerflow.runtime.user_context import reset_current_user, set_current_user
|
||||
from deerflow.skills import curator_log
|
||||
|
||||
# 总开关 + curator 子开关:任一关闭则直接跳过。手动触发 /api/curator/run 也经由
|
||||
# 本函数,因此手动触发同样受总开关与 curator.enabled 约束。
|
||||
if not _skill_evolution_enabled(user_id):
|
||||
logger.info("Curator[%s]: 技能进化总开关关闭,跳过", user_id)
|
||||
return
|
||||
curator_cfg = _get_curator_config(user_id)
|
||||
if curator_cfg is None or not getattr(curator_cfg, "enabled", False):
|
||||
logger.info("Curator[%s]: curator 未启用,跳过", user_id)
|
||||
return
|
||||
|
||||
# 运行锁:防止同一用户的 curator 并发(定时器 vs 手动触发、连点手动触发)。抢不到
|
||||
# 锁说明已有一轮在跑,直接跳过本次触发。
|
||||
if not _try_acquire_run_lock(user_id, datetime.now(UTC)):
|
||||
logger.info("Curator[%s]: 已有 curator 正在运行,跳过本次触发", user_id)
|
||||
curator_log.record("info", "已有 curator 正在运行,跳过本次触发", user_id=user_id)
|
||||
return
|
||||
|
||||
# Bind the per-thread user context for the entire curator run so every
|
||||
# downstream tool / repository call resolves the correct user_id.
|
||||
token = set_current_user(_CuratorUser(user_id))
|
||||
@@ -765,13 +1093,16 @@ def _curator_thread_target(user_id: str) -> None:
|
||||
curator_log.record("error", f"策展后台任务失败:{exc}", user_id=user_id)
|
||||
finally:
|
||||
reset_current_user(token)
|
||||
# 无论成功或异常,必定释放运行锁。
|
||||
_release_run_lock(user_id)
|
||||
|
||||
|
||||
async def maybe_run_curator() -> None:
|
||||
"""Async startup entry — kicks off a thread per user whose interval elapsed.
|
||||
"""周期调度入口 —— 给每个 interval 已到期的用户起一个后台策展线程。
|
||||
|
||||
Skips users with no DB-tracked entries (a fresh deployment) and never
|
||||
blocks lifespan startup on failure.
|
||||
由 app 的 curator 调度器周期性调用(也用于手动触发)。无候选技能的用户,其
|
||||
LLM 评审会在 run_curator_review 内被跳过 —— 只做不耗费 LLM 的年龄流转。
|
||||
任何失败都不会阻断调用方。
|
||||
"""
|
||||
try:
|
||||
from sqlalchemy import select
|
||||
@@ -789,20 +1120,42 @@ async def maybe_run_curator() -> None:
|
||||
logger.exception("Curator: failed to enumerate users at startup")
|
||||
return
|
||||
|
||||
due_users: list[str] = []
|
||||
for user_id in user_ids:
|
||||
try:
|
||||
if not should_run_now(user_id):
|
||||
continue
|
||||
logger.info("Curator[%s]: interval elapsed, spawning background thread", user_id)
|
||||
thread = threading.Thread(
|
||||
target=_curator_thread_target,
|
||||
args=(user_id,),
|
||||
daemon=True,
|
||||
name=f"skill-curator-{user_id}",
|
||||
)
|
||||
thread.start()
|
||||
if should_run_now(user_id):
|
||||
due_users.append(user_id)
|
||||
except Exception:
|
||||
logger.exception("Curator: failed to spawn for user %s", user_id)
|
||||
logger.exception("Curator: should_run_now check failed for user %s", user_id)
|
||||
_dispatch_curator_runs(due_users)
|
||||
|
||||
|
||||
def _dispatch_curator_runs(due_users: list[str]) -> None:
|
||||
"""用线程池(并发上限 ``_CURATOR_MAX_CONCURRENCY``)派发到期用户的 curator 执行。
|
||||
|
||||
到期用户排队消费,而非一次性全部起线程齐调 LLM —— 防首次部署所有用户同时到期时
|
||||
的惊群。``shutdown(wait=False)``:提交后立即返回,线程池在所有任务跑完后自行回收。
|
||||
抽成独立函数便于对并发上限做单元测试。
|
||||
"""
|
||||
if not due_users:
|
||||
return
|
||||
|
||||
logger.info(
|
||||
"Curator: %d 个用户到期,以并发上限 %d 派发执行",
|
||||
len(due_users),
|
||||
_CURATOR_MAX_CONCURRENCY,
|
||||
)
|
||||
executor = concurrent.futures.ThreadPoolExecutor(
|
||||
max_workers=_CURATOR_MAX_CONCURRENCY,
|
||||
thread_name_prefix="skill-curator",
|
||||
)
|
||||
try:
|
||||
for user_id in due_users:
|
||||
executor.submit(_curator_thread_target, user_id)
|
||||
except Exception:
|
||||
logger.exception("Curator: failed to submit curator tasks")
|
||||
finally:
|
||||
executor.shutdown(wait=False)
|
||||
|
||||
|
||||
def get_curator_status(user_id: str) -> dict:
|
||||
@@ -819,7 +1172,7 @@ def get_curator_status(user_id: str) -> dict:
|
||||
s = entry.get("state", STATE_ACTIVE)
|
||||
counts[s] = counts.get(s, 0) + 1
|
||||
|
||||
cfg = _get_curator_config()
|
||||
cfg = _get_curator_config(user_id)
|
||||
return {
|
||||
"enabled": cfg is not None and getattr(cfg, "enabled", True),
|
||||
"run_count": state.get("run_count", 0),
|
||||
@@ -832,4 +1185,8 @@ def get_curator_status(user_id: str) -> dict:
|
||||
"stale": counts[STATE_STALE],
|
||||
"archived": counts[STATE_ARCHIVED],
|
||||
},
|
||||
# 上次运行的两份记录:LLM 自述的建议 + 实际发生的工具调用,供前端对照展示。
|
||||
"last_consolidation_suggestions": state.get("last_consolidation_suggestions")
|
||||
or {"consolidations": [], "prunings": []},
|
||||
"last_actual_actions": state.get("last_actual_actions") or [],
|
||||
}
|
||||
+4
-2
@@ -50,8 +50,10 @@ async def scan_skill_content(content: str, *, executable: bool = False, location
|
||||
|
||||
try:
|
||||
config = app_config or get_app_config()
|
||||
model_name = config.skill_evolution.moderation_model_name
|
||||
model = create_chat_model(name=model_name, thinking_enabled=False, app_config=config) if model_name else create_chat_model(thinking_enabled=False, app_config=config)
|
||||
# 安全审核复用 curator 的评审模型配置(skill_evolution.curator.model_name):
|
||||
# 二者用同一个模型。未配置时 model_name 为 None,create_chat_model 回退主模型。
|
||||
model_name = config.skill_evolution.curator.model_name
|
||||
model = create_chat_model(name=model_name, thinking_enabled=False, app_config=config)
|
||||
response = await model.ainvoke(
|
||||
[
|
||||
{"role": "system", "content": rubric},
|
||||
|
||||
@@ -6,6 +6,7 @@ import json
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
@@ -13,6 +14,11 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
_USAGE_FILENAME = ".usage.json"
|
||||
|
||||
# 串行化对 .usage.json 的"读-改-写",避免并发(尤其 curator 多线程)丢更新。
|
||||
# .usage.json 是全局单文件;此锁覆盖进程内并发。leader 选举已使 curator 只在
|
||||
# 单进程跑,故进程内锁即可保证状态流转不丢。
|
||||
_usage_lock = threading.Lock()
|
||||
|
||||
STATE_ACTIVE = "active"
|
||||
STATE_STALE = "stale"
|
||||
STATE_ARCHIVED = "archived"
|
||||
@@ -75,13 +81,20 @@ def _now_iso() -> str:
|
||||
return datetime.now(UTC).isoformat()
|
||||
|
||||
|
||||
def _update_entry(name: str, updates: dict) -> None:
|
||||
def _update_entry(name: str, updates) -> None:
|
||||
"""更新一个 usage 条目。``updates`` 可以是 dict,或 ``(entry) -> dict`` 回调。
|
||||
|
||||
用回调形式可让"读当前值 -> 算增量"也落在锁内(例如计数自增),
|
||||
避免并发下的 lost update。
|
||||
"""
|
||||
try:
|
||||
data = _read_all()
|
||||
entry = {**_DEFAULT_ENTRY, **data.get(name, {})}
|
||||
entry.update(updates)
|
||||
data[name] = entry
|
||||
_write_all(data)
|
||||
with _usage_lock:
|
||||
data = _read_all()
|
||||
entry = {**_DEFAULT_ENTRY, **data.get(name, {})}
|
||||
resolved = updates(entry) if callable(updates) else updates
|
||||
entry.update(resolved)
|
||||
data[name] = entry
|
||||
_write_all(data)
|
||||
except Exception:
|
||||
logger.debug("Failed to update usage entry for %r", name, exc_info=True)
|
||||
|
||||
@@ -103,18 +116,16 @@ def agent_created_entries() -> dict:
|
||||
|
||||
|
||||
def bump_view(name: str) -> None:
|
||||
entry = get_entry(name)
|
||||
_update_entry(name, {"view_count": entry.get("view_count", 0) + 1, "last_viewed_at": _now_iso()})
|
||||
# 用回调:读当前计数 + 自增在同一把锁内完成,避免并发丢计数。
|
||||
_update_entry(name, lambda e: {"view_count": e.get("view_count", 0) + 1, "last_viewed_at": _now_iso()})
|
||||
|
||||
|
||||
def bump_use(name: str) -> None:
|
||||
entry = get_entry(name)
|
||||
_update_entry(name, {"use_count": entry.get("use_count", 0) + 1, "last_used_at": _now_iso()})
|
||||
_update_entry(name, lambda e: {"use_count": e.get("use_count", 0) + 1, "last_used_at": _now_iso()})
|
||||
|
||||
|
||||
def bump_patch(name: str) -> None:
|
||||
entry = get_entry(name)
|
||||
_update_entry(name, {"patch_count": entry.get("patch_count", 0) + 1, "last_patched_at": _now_iso()})
|
||||
_update_entry(name, lambda e: {"patch_count": e.get("patch_count", 0) + 1, "last_patched_at": _now_iso()})
|
||||
|
||||
|
||||
def set_state(name: str, state: str) -> None:
|
||||
@@ -132,9 +143,10 @@ def set_pinned(name: str, pinned: bool) -> None:
|
||||
|
||||
def forget(name: str) -> None:
|
||||
try:
|
||||
data = _read_all()
|
||||
data.pop(name, None)
|
||||
_write_all(data)
|
||||
with _usage_lock:
|
||||
data = _read_all()
|
||||
data.pop(name, None)
|
||||
_write_all(data)
|
||||
except Exception:
|
||||
logger.debug("Failed to forget usage entry for %r", name, exc_info=True)
|
||||
|
||||
|
||||
+84
-5
@@ -59,6 +59,66 @@ def _load_agent_skill_whitelist(agent_id: str | None) -> set[str] | None:
|
||||
return None
|
||||
|
||||
|
||||
def _resolve_effective_user_id() -> str | None:
|
||||
"""当前请求/线程绑定的用户 id;解析失败时返回 None。"""
|
||||
try:
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
|
||||
return get_effective_user_id()
|
||||
except Exception:
|
||||
logger.debug("Failed to resolve effective user id for skill_list", exc_info=True)
|
||||
return None
|
||||
|
||||
|
||||
def _run_coro_blocking(coro):
|
||||
"""在同步工具里把一个协程跑到完成。
|
||||
|
||||
``skill_list`` 是同步 ``@tool``:LangChain 通常在线程池里执行它(无运行中的
|
||||
事件循环),但 curator 评审会在已有事件循环的线程里调用它。两种情况都要能
|
||||
拿到 ``SkillStore`` 的异步查询结果 —— 检测到运行中的事件循环就转交独立线程
|
||||
执行 ``asyncio.run``,否则直接跑。
|
||||
"""
|
||||
import asyncio
|
||||
|
||||
try:
|
||||
asyncio.get_running_loop()
|
||||
in_loop = True
|
||||
except RuntimeError:
|
||||
in_loop = False
|
||||
|
||||
if in_loop:
|
||||
import concurrent.futures
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
|
||||
return pool.submit(asyncio.run, coro).result()
|
||||
return asyncio.run(coro)
|
||||
|
||||
|
||||
def _load_owned_skill_names(user_id: str | None) -> set[str] | None:
|
||||
"""返回 ``owner_user_id == user_id`` 的自定义技能名集合。
|
||||
|
||||
用于默认 lead agent 的可见性过滤:只算"用户自己拥有的"自定义技能,不含
|
||||
其他用户发布的。返回 ``None`` 表示 ``SkillStore`` 不可用(未配置数据库
|
||||
等)—— 调用方应降级为"放行全部自定义技能",避免离线无库部署下技能凭空
|
||||
消失。
|
||||
"""
|
||||
if not user_id:
|
||||
return None
|
||||
try:
|
||||
from deerflow.persistence.engine import get_session_factory
|
||||
from deerflow.persistence.skills import make_skill_store
|
||||
|
||||
factory = get_session_factory()
|
||||
if factory is None:
|
||||
return None
|
||||
store = make_skill_store(factory)
|
||||
records = _run_coro_blocking(store.list_visible(user_id))
|
||||
return {r["name"] for r in records if r.get("owner_user_id") == user_id}
|
||||
except Exception:
|
||||
logger.debug("Failed to load owned skill names for skill_list filter", exc_info=True)
|
||||
return None
|
||||
|
||||
|
||||
@tool("skill_view")
|
||||
def skill_view_tool(name: str) -> str:
|
||||
"""Read the full SKILL.md content of a skill by name.
|
||||
@@ -106,28 +166,47 @@ def skill_list_tool(runtime: ToolRuntime) -> str:
|
||||
under the ``skills:`` field — the agent only sees what it was wired to
|
||||
use.
|
||||
|
||||
For the **default lead agent** (no agent_id in runtime), there is no
|
||||
per-agent whitelist and every visible skill is returned.
|
||||
For the **default lead agent** (no agent_id in runtime), the result is
|
||||
filtered to non-archived skills that are either built-in (public) or
|
||||
owned by the current user. Other users' published custom skills and
|
||||
archived skills are excluded.
|
||||
|
||||
Returns a JSON array with fields: name, description, category, enabled,
|
||||
state, use_count, view_count, patch_count, last_activity_at, pinned,
|
||||
source.
|
||||
"""
|
||||
from deerflow.skills.storage import get_or_new_skill_storage
|
||||
from deerflow.skills.usage import all_entries, latest_activity_at
|
||||
from deerflow.skills.types import SkillCategory
|
||||
from deerflow.skills.usage import STATE_ARCHIVED, all_entries, latest_activity_at
|
||||
|
||||
agent_id = _resolve_active_agent_id(runtime)
|
||||
allowed = _load_agent_skill_whitelist(agent_id)
|
||||
|
||||
# 默认 lead agent 没有 per-agent 白名单 —— 改为只暴露"未归档"且"内置或属于
|
||||
# 当前用户"的技能。owned 为 None 表示 SkillStore 不可用,降级放行全部自定义
|
||||
# 技能(见 _load_owned_skill_names)。
|
||||
owned: set[str] | None = None
|
||||
if allowed is None:
|
||||
owned = _load_owned_skill_names(_resolve_effective_user_id())
|
||||
|
||||
try:
|
||||
storage = get_or_new_skill_storage()
|
||||
skills = storage.load_skills(enabled_only=False)
|
||||
usage = all_entries()
|
||||
result = []
|
||||
for skill in skills:
|
||||
if allowed is not None and skill.name not in allowed:
|
||||
continue
|
||||
entry = usage.get(skill.name, {})
|
||||
if allowed is not None:
|
||||
# 自定义 agent:严格按 config.yaml 的 skills 白名单过滤。
|
||||
if skill.name not in allowed:
|
||||
continue
|
||||
else:
|
||||
# 默认 agent:跳过已归档技能。
|
||||
if entry.get("state") == STATE_ARCHIVED:
|
||||
continue
|
||||
# 自定义技能仅当属于当前用户时可见;内置 public 技能始终可见。
|
||||
if skill.category == SkillCategory.CUSTOM and owned is not None and skill.name not in owned:
|
||||
continue
|
||||
result.append({
|
||||
"name": skill.name,
|
||||
"description": skill.description,
|
||||
|
||||
@@ -84,12 +84,26 @@ def get_available_tools(
|
||||
builtin_tools = BUILTIN_TOOLS.copy()
|
||||
if not is_scheduled_run:
|
||||
builtin_tools.append(scheduled_task_tool)
|
||||
skill_evolution_config = getattr(config, "skill_evolution", None)
|
||||
# skill_list / skill_view 是只读工具(发现、查看技能),与"自我进化"写能力无关,
|
||||
# 始终提供给 agent。
|
||||
from deerflow.tools.builtins.skill_tools import skill_list_tool, skill_view_tool
|
||||
|
||||
builtin_tools.extend([skill_list_tool, skill_view_tool])
|
||||
|
||||
# skill_manage 能创建/修改/删除技能,属于"自我进化"写能力,按 skill_evolution.enabled
|
||||
# 门控。该开关用户可在 per-user 配置里覆盖,所以按"生效配置(含 per-user 覆盖)"判断 ——
|
||||
# Gateway 每次 run 重建 agent,用户改了配置下次 run 即生效;解析失败退回全局 config.yaml。
|
||||
try:
|
||||
from deerflow.config.skill_evolution_runtime import load_skill_evolution_config_for_user
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
|
||||
skill_evolution_config = load_skill_evolution_config_for_user(get_effective_user_id())
|
||||
except Exception: # noqa: BLE001 - 解析失败不能阻断 agent 构建
|
||||
skill_evolution_config = getattr(config, "skill_evolution", None)
|
||||
if getattr(skill_evolution_config, "enabled", False):
|
||||
from deerflow.tools.builtins.skill_tools import skill_list_tool, skill_view_tool
|
||||
from deerflow.tools.skill_manage_tool import skill_manage_tool
|
||||
|
||||
builtin_tools.extend([skill_list_tool, skill_view_tool, skill_manage_tool])
|
||||
builtin_tools.append(skill_manage_tool)
|
||||
|
||||
# Add subagent tools only if enabled via runtime parameter
|
||||
if subagent_enabled:
|
||||
|
||||
@@ -25,12 +25,12 @@ def test_load_overrides_returns_none_when_missing(overrides_path):
|
||||
def test_save_and_load_overrides_roundtrip(overrides_path):
|
||||
payload = {
|
||||
"enabled": True,
|
||||
"moderation_model_name": "moderation-mini",
|
||||
"curator": {
|
||||
"enabled": True,
|
||||
"interval_hours": 12,
|
||||
"stale_after_days": 7,
|
||||
"archive_after_days": 30,
|
||||
"model_name": "curator-mini",
|
||||
},
|
||||
}
|
||||
written = skill_evolution_runtime.save_overrides(payload)
|
||||
|
||||
@@ -501,7 +501,8 @@ def test_curator_skips_published_skill(isolated_paths, monkeypatch) -> None:
|
||||
|
||||
monkeypatch.setattr(curator, "_list_user_owned_skill_names", lambda uid: {"public-tool"})
|
||||
monkeypatch.setattr(curator, "_collect_referenced_skill_names", lambda: set())
|
||||
monkeypatch.setattr(curator, "_is_skill_published", lambda name: name == "public-tool")
|
||||
# curator 现在用批量接口一次性查 published 集合,而非逐技能 _is_skill_published。
|
||||
monkeypatch.setattr(curator, "_list_published_skill_names", lambda: {"public-tool"})
|
||||
_isolated_skills_root(isolated_paths, monkeypatch)
|
||||
_seed_old_agent_skill("public-tool")
|
||||
|
||||
|
||||
Generated
+2692
-2692
File diff suppressed because it is too large.
Load diff
Reference in new issue
Block a user