# Conflicts:
#	frontend-web/src/pages/WorkspaceRoutes.tsx
#	offline-backend-20260512/backend/packages/harness/deerflow/persistence/models/__init__.py
This commit is contained in:
吉鑫升 committed 2026-05-15 11:21:48 +08:00
commit aeba15eb16
88 files changed
+13824 -3572

No files matched your search

@@ -191,14 +191,23 @@ def _build_skill_evolution_section(skill_evolution_enabled: bool) -> str:
return ""
return """
## Skill Self-Evolution
After completing a task, consider creating or updating a skill when:
- The task required 5+ tool calls to resolve
- You overcame non-obvious errors or pitfalls
- The user corrected your approach and the corrected version worked
- You discovered a non-trivial, recurring workflow
If you used a skill and encountered issues not covered by it, patch it immediately.
Prefer patch over edit. Before creating a new skill, confirm with the user first.
Skip simple one-off tasks.
You have access to `skill_manage` (plus `skill_list` / `skill_view`).
After completing a non-trivial task — 5+ tool calls to resolve, recovery from
a non-obvious error, a corrected approach that worked, or a recurring
workflow — **offer to save the approach as a reusable skill** so you can
replay it next time. Briefly propose a hyphen-case name and a one-line
description, then ask the user to confirm before calling
`skill_manage(action="create", ...)`. Capture the generalised steps, not the
user's specific data, paths, or credentials.
When using an existing skill and finding it outdated, incomplete, or wrong,
**patch it immediately** with `skill_manage(action="patch")` — don't wait to
be asked. Skills that aren't maintained become liabilities.
Skip the offer for one-off queries, casual chat, or short single-tool
answers — and never create a skill silently without user confirmation.
"""
@@ -106,7 +106,7 @@ class ThreadDataMiddleware(AgentMiddleware[ThreadDataMiddlewareState]):
messages[-1] = HumanMessage(
content=last_message.content,
id=last_message.id,
name=last_message.name or "user-input",
name=last_message.name,
additional_kwargs={**last_message.additional_kwargs, "run_id": runtime.context.get("run_id"), "timestamp": datetime.now(UTC).isoformat()},
)
@@ -20,6 +20,8 @@ from deerflow.config.run_events_config import RunEventsConfig
from deerflow.config.runtime_paths import existing_project_file
from deerflow.config.sandbox_config import SandboxConfig
from deerflow.config.skill_evolution_config import SkillEvolutionConfig
from deerflow.config.skill_evolution_runtime import get_overrides_mtime as _get_skill_evolution_overrides_mtime
from deerflow.config.skill_evolution_runtime import load_overrides as _load_skill_evolution_overrides
from deerflow.config.skills_config import SkillsConfig
from deerflow.config.stream_bridge_config import StreamBridgeConfig, load_stream_bridge_config_from_dict
from deerflow.config.subagents_config import SubagentsAppConfig, load_subagents_config_from_dict
@@ -54,6 +56,22 @@ def _legacy_config_candidates() -> tuple[Path, ...]:
return (backend_dir / "config.yaml", repo_root / "config.yaml")
def _deep_merge_dict(base: dict[str, Any], overrides: dict[str, Any]) -> dict[str, Any]:
"""Deep-merge ``overrides`` onto a copy of ``base`` and return the result.
Nested dicts are merged recursively; any other value type in ``overrides``
replaces the corresponding value in ``base`` outright.
"""
merged: dict[str, Any] = dict(base)
for key, value in overrides.items():
existing = merged.get(key)
if isinstance(existing, dict) and isinstance(value, dict):
merged[key] = _deep_merge_dict(existing, value)
else:
merged[key] = value
return merged
def logging_level_from_config(name: str | None) -> int:
"""Map ``config.yaml`` ``log_level`` string to a :mod:`logging` level constant."""
mapping = logging.getLevelNamesMapping()
@@ -204,6 +222,15 @@ class AppConfig(BaseModel):
extensions_config = ExtensionsConfig.from_file()
config_data["extensions"] = extensions_config.model_dump()
# Apply skill_evolution runtime overrides from skill_evolution.json
# (file is optional; load_overrides returns None if missing/invalid)
overrides = _load_skill_evolution_overrides()
if overrides:
yaml_section = config_data.get("skill_evolution") or {}
if not isinstance(yaml_section, dict):
yaml_section = {}
config_data["skill_evolution"] = _deep_merge_dict(yaml_section, overrides)
result = cls.model_validate(config_data)
return result
@@ -329,6 +356,7 @@ class AppConfig(BaseModel):
_app_config: AppConfig | None = None
_app_config_path: Path | None = None
_app_config_mtime: float | None = None
_app_config_skill_evolution_overrides_mtime: float | None = None
_app_config_is_custom = False
_current_app_config: ContextVar[AppConfig | None] = ContextVar("deerflow_current_app_config", default=None)
_current_app_config_stack: ContextVar[tuple[AppConfig | None, ...]] = ContextVar("deerflow_current_app_config_stack", default=())
@@ -344,12 +372,13 @@ def _get_config_mtime(config_path: Path) -> float | None:
def _load_and_cache_app_config(config_path: str | None = None) -> AppConfig:
"""Load config from disk and refresh cache metadata."""
global _app_config, _app_config_path, _app_config_mtime, _app_config_is_custom
global _app_config, _app_config_path, _app_config_mtime, _app_config_skill_evolution_overrides_mtime, _app_config_is_custom
resolved_path = AppConfig.resolve_config_path(config_path)
_app_config = AppConfig.from_file(str(resolved_path))
_app_config_path = resolved_path
_app_config_mtime = _get_config_mtime(resolved_path)
_app_config_skill_evolution_overrides_mtime = _get_skill_evolution_overrides_mtime()
_app_config_is_custom = False
return _app_config
@@ -373,8 +402,14 @@ def get_app_config() -> AppConfig:
resolved_path = AppConfig.resolve_config_path()
current_mtime = _get_config_mtime(resolved_path)
current_overrides_mtime = _get_skill_evolution_overrides_mtime()
should_reload = _app_config is None or _app_config_path != resolved_path or _app_config_mtime != current_mtime
should_reload = (
_app_config is None
or _app_config_path != resolved_path
or _app_config_mtime != current_mtime
or _app_config_skill_evolution_overrides_mtime != current_overrides_mtime
)
if should_reload:
if _app_config_path == resolved_path and _app_config_mtime is not None and current_mtime is not None and _app_config_mtime != current_mtime:
logger.info(
@@ -382,6 +417,12 @@ def get_app_config() -> AppConfig:
_app_config_mtime,
current_mtime,
)
elif _app_config_skill_evolution_overrides_mtime != current_overrides_mtime:
logger.info(
"skill_evolution.json overrides changed (mtime: %s -> %s), reloading AppConfig",
_app_config_skill_evolution_overrides_mtime,
current_overrides_mtime,
)
_load_and_cache_app_config(str(resolved_path))
return _app_config
@@ -409,10 +450,11 @@ def reset_app_config() -> None:
`get_app_config()` to reload from file. Useful for testing
or when switching between different configurations.
"""
global _app_config, _app_config_path, _app_config_mtime, _app_config_is_custom
global _app_config, _app_config_path, _app_config_mtime, _app_config_skill_evolution_overrides_mtime, _app_config_is_custom
_app_config = None
_app_config_path = None
_app_config_mtime = None
_app_config_skill_evolution_overrides_mtime = None
_app_config_is_custom = False
@@ -424,10 +466,11 @@ def set_app_config(config: AppConfig) -> None:
Args:
config: The AppConfig instance to use.
"""
global _app_config, _app_config_path, _app_config_mtime, _app_config_is_custom
global _app_config, _app_config_path, _app_config_mtime, _app_config_skill_evolution_overrides_mtime, _app_config_is_custom
_app_config = config
_app_config_path = None
_app_config_mtime = None
_app_config_skill_evolution_overrides_mtime = None
_app_config_is_custom = True
@@ -1,6 +1,15 @@
from pydantic import BaseModel, Field
class CuratorConfig(BaseModel):
"""Configuration for the background skill curator."""
enabled: bool = Field(default=True, description="Whether the curator runs automatically.")
interval_hours: int = Field(default=168, description="Hours between curator runs (default: 7 days).")
stale_after_days: int = Field(default=30, description="Days of inactivity before a skill is marked stale.")
archive_after_days: int = Field(default=90, description="Days of inactivity before a stale skill is archived.")
class SkillEvolutionConfig(BaseModel):
"""Configuration for agent-managed skill evolution."""
@@ -12,3 +21,7 @@ class SkillEvolutionConfig(BaseModel):
default=None,
description="Optional model name for skill security moderation. Defaults to the primary chat model.",
)
curator: CuratorConfig = Field(
default_factory=CuratorConfig,
description="Configuration for the background skill curator.",
)
@@ -0,0 +1,216 @@
"""Runtime overrides for `skill_evolution` config, stored in skill_evolution.json.
Two scopes exist:
* **Global** (back-compat): ``backend/skill_evolution.json`` deep-merged onto
``config.yaml.skill_evolution`` at AppConfig load time. Used as a fallback
for users who never set a personal override.
* **Per-user**: ``.deer-flow/users/{user_id}/skill_evolution.json``. Read by
the curator on each run and by the ``GET/PUT /api/curator/config`` route
with the active user id. Curator merges this on top of the global config
via :func:`load_curator_config_for_user`.
The file is optional in both scopes; absence (or a malformed payload) is
treated as "no overrides" so the system keeps the YAML-only behaviour.
"""
from __future__ import annotations
import json
import logging
import os
import tempfile
from pathlib import Path
from typing import Any
from deerflow.config.runtime_paths import existing_project_file
logger = logging.getLogger(__name__)
_FILENAME = "skill_evolution.json"
def resolve_path() -> Path:
"""Resolve the on-disk location of ``skill_evolution.json``.
Priority:
1. ``DEER_FLOW_SKILL_EVOLUTION_CONFIG_PATH`` environment variable
2. ``skill_evolution.json`` in the project root (current and parents)
3. Legacy ``backend/`` or repo-root sibling locations
4. Falls back to ``backend/skill_evolution.json`` (does not need to exist)
"""
override = os.getenv("DEER_FLOW_SKILL_EVOLUTION_CONFIG_PATH")
if override:
return Path(override)
project_file = existing_project_file((_FILENAME,))
if project_file is not None:
return project_file
backend_dir = Path(__file__).resolve().parents[4]
repo_root = backend_dir.parent
for candidate in (backend_dir / _FILENAME, repo_root / _FILENAME):
if candidate.exists():
return candidate
return backend_dir / _FILENAME
def load_overrides() -> dict[str, Any] | None:
"""Read overrides from disk.
Returns ``None`` when the file does not exist, is empty, contains
non-dict JSON, or fails to parse. Errors are logged at WARNING level
so production stays silent in the common "no overrides" case.
"""
path = resolve_path()
if not path.exists():
return None
try:
raw = path.read_text(encoding="utf-8").strip()
if not raw:
return None
data = json.loads(raw)
except OSError as exc:
logger.warning("Failed to read %s: %s", path, exc)
return None
except json.JSONDecodeError as exc:
logger.warning("Skill evolution overrides at %s are not valid JSON: %s", path, exc)
return None
if not isinstance(data, dict):
logger.warning("Skill evolution overrides at %s must be a JSON object, got %s", path, type(data).__name__)
return None
return data
def save_overrides(payload: dict[str, Any]) -> Path:
"""Atomically persist ``payload`` to ``skill_evolution.json``.
The caller is responsible for shape validation (e.g. via the
``SkillEvolutionConfig`` Pydantic model). Returns the path that was
written so callers can log it.
"""
if not isinstance(payload, dict):
raise TypeError("skill_evolution overrides payload must be a dict")
path = resolve_path()
path.parent.mkdir(parents=True, exist_ok=True)
fd, tmp_path_str = tempfile.mkstemp(prefix=".skill-evolution-", suffix=".tmp", dir=str(path.parent))
tmp_path = Path(tmp_path_str)
try:
with os.fdopen(fd, "w", encoding="utf-8") as f:
json.dump(payload, f, indent=2, ensure_ascii=False)
os.replace(tmp_path, path)
except Exception:
tmp_path.unlink(missing_ok=True)
raise
logger.info("Wrote skill_evolution overrides to %s", path)
return path
def get_overrides_mtime() -> float | None:
"""Return the mtime of ``skill_evolution.json`` if it exists, else None.
Used by the AppConfig cache to decide whether to reload — when the
file appears, disappears, or is modified, the mtime tuple changes
and ``get_app_config()`` rebuilds.
"""
path = resolve_path()
try:
return path.stat().st_mtime
except OSError:
return None
# ---------------------------------------------------------------------------
# Per-user overrides
# ---------------------------------------------------------------------------
def resolve_user_path(user_id: str) -> Path:
"""Return ``{base_dir}/users/{user_id}/skill_evolution.json``."""
from deerflow.config.paths import get_paths
return get_paths().user_dir(user_id) / _FILENAME
def load_user_overrides(user_id: str) -> dict[str, Any] | None:
"""Read the per-user overrides file for ``user_id`` (None if missing/invalid)."""
path = resolve_user_path(user_id)
if not path.exists():
return None
try:
raw = path.read_text(encoding="utf-8").strip()
if not raw:
return None
data = json.loads(raw)
except OSError as exc:
logger.warning("Failed to read %s: %s", path, exc)
return None
except json.JSONDecodeError as exc:
logger.warning("User skill_evolution overrides at %s are not valid JSON: %s", path, exc)
return None
if not isinstance(data, dict):
logger.warning("User skill_evolution overrides at %s must be a JSON object", path)
return None
return data
def save_user_overrides(user_id: str, payload: dict[str, Any]) -> Path:
"""Atomically persist ``payload`` to the per-user overrides file."""
if not isinstance(payload, dict):
raise TypeError("skill_evolution overrides payload must be a dict")
path = resolve_user_path(user_id)
path.parent.mkdir(parents=True, exist_ok=True)
fd, tmp_path_str = tempfile.mkstemp(prefix=".skill-evolution-", suffix=".tmp", dir=str(path.parent))
tmp_path = Path(tmp_path_str)
try:
with os.fdopen(fd, "w", encoding="utf-8") as f:
json.dump(payload, f, indent=2, ensure_ascii=False)
os.replace(tmp_path, path)
except Exception:
tmp_path.unlink(missing_ok=True)
raise
logger.info("Wrote per-user skill_evolution overrides to %s", path)
return path
def get_user_overrides_mtime(user_id: str) -> float | None:
path = resolve_user_path(user_id)
try:
return path.stat().st_mtime
except OSError:
return None
def load_skill_evolution_config_for_user(user_id: str | None):
"""Return a ``SkillEvolutionConfig`` instance with per-user overrides merged
on top of the global AppConfig defaults.
Used by the curator + the per-user ``GET/PUT /api/curator/config`` route.
When ``user_id`` is None or the per-user file is absent, returns the global
config unchanged.
"""
from deerflow.config import get_app_config
from deerflow.config.skill_evolution_config import SkillEvolutionConfig
base = get_app_config().skill_evolution
if not user_id:
return base
overrides = load_user_overrides(user_id)
if not overrides:
return base
merged = base.model_dump()
for key, value in overrides.items():
existing = merged.get(key)
if isinstance(existing, dict) and isinstance(value, dict):
merged[key] = {**existing, **value}
else:
merged[key] = value
try:
return SkillEvolutionConfig.model_validate(merged)
except Exception:
logger.warning("Per-user skill_evolution overrides for %s failed validation, ignoring", user_id, exc_info=True)
return base
@@ -56,3 +56,8 @@ class AgentStore(ABC):
any user identity and returns ``False`` if the target is not a built-in.
"""
raise NotImplementedError
@abstractmethod
async def count_owned(self, user_id: str) -> int:
"""Return the number of agents owned by ``user_id`` (user_id IS NOT NULL)."""
raise NotImplementedError
@@ -114,3 +114,6 @@ class MemoryAgentStore(AgentStore):
return False
self._agents.pop(agent_id, None)
return True
async def count_owned(self, user_id: str) -> int:
return sum(1 for a in self._agents.values() if a.get("user_id") == user_id)
@@ -5,7 +5,7 @@ from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import or_, select
from sqlalchemy import func, or_, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.agents.base import AgentStore
@@ -137,3 +137,8 @@ class AgentRepository(AgentStore):
await session.delete(row)
await session.commit()
return True
async def count_owned(self, user_id: str) -> int:
stmt = select(func.count()).select_from(AgentRow).where(AgentRow.user_id == user_id)
async with self._sf() as session:
return await session.scalar(stmt) or 0
@@ -19,6 +19,7 @@ from deerflow.persistence.agents.model import AgentRow
from deerflow.persistence.feedback.model import FeedbackRow
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
from deerflow.persistence.models.run_event import RunEventRow
from deerflow.persistence.notifications.model import NotificationRow
from deerflow.persistence.recommended_questions.model import RecommendedQuestionRow
from deerflow.persistence.run.model import RunRow
from deerflow.persistence.scheduled_tasks.model import (
@@ -28,6 +29,7 @@ from deerflow.persistence.scheduled_tasks.model import (
ScheduledTaskSubscriptionRow,
SchedulerThreadRow,
)
from deerflow.persistence.skills.model import SkillRow
from deerflow.persistence.thread_meta.model import ThreadMetaRow
from deerflow.persistence.user.model import UserRow
@@ -35,6 +37,7 @@ __all__ = [
"FeedbackRow",
"AgentRow",
"LlmCallMetricRow",
"NotificationRow",
"RecommendedQuestionRow",
"RunEventRow",
"RunRow",
@@ -43,6 +46,7 @@ __all__ = [
"ScheduledTaskRunRow",
"ScheduledTaskSubscriptionRow",
"SchedulerThreadRow",
"SkillRow",
"ThreadMetaRow",
"UserRow",
]
@@ -0,0 +1,26 @@
"""Persistence layer for per-user in-app notifications."""
from deerflow.persistence.notifications.base import NotificationStore
from deerflow.persistence.notifications.memory import MemoryNotificationStore
from deerflow.persistence.notifications.sender import (
get_notification_store,
notify,
set_notification_store,
)
from deerflow.persistence.notifications.sql import NotificationRepository
__all__ = [
"NotificationStore",
"MemoryNotificationStore",
"NotificationRepository",
"notify",
"set_notification_store",
"get_notification_store",
"make_notification_store",
]
def make_notification_store(session_factory):
if session_factory is None:
return MemoryNotificationStore()
return NotificationRepository(session_factory)
@@ -0,0 +1,42 @@
"""Abstract persistence API for in-app notifications."""
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any
class NotificationStore(ABC):
@abstractmethod
async def create(self, data: dict[str, Any]) -> dict[str, Any]:
"""Insert a notification record. ``data`` must include ``id``, ``user_id``, ``type``."""
raise NotImplementedError
@abstractmethod
async def list_for_user(
self,
user_id: str,
*,
unread_only: bool = False,
limit: int = 50,
) -> list[dict[str, Any]]:
"""Return newest-first records for ``user_id``."""
raise NotImplementedError
@abstractmethod
async def unread_count(self, user_id: str) -> int:
raise NotImplementedError
@abstractmethod
async def mark_read(self, notification_id: str, user_id: str) -> bool:
"""Mark a single notification as read; returns False if not owned by user."""
raise NotImplementedError
@abstractmethod
async def mark_all_read(self, user_id: str) -> int:
"""Mark every unread notification of ``user_id`` as read; returns count updated."""
raise NotImplementedError
@abstractmethod
async def delete(self, notification_id: str, user_id: str) -> bool:
raise NotImplementedError
@@ -0,0 +1,70 @@
"""In-memory ``NotificationStore`` implementation for tests."""
from __future__ import annotations
import copy
from datetime import UTC, datetime
from typing import Any
from deerflow.persistence.notifications.base import NotificationStore
def _now_iso() -> str:
return datetime.now(UTC).isoformat()
class MemoryNotificationStore(NotificationStore):
def __init__(self) -> None:
self._items: dict[str, dict[str, Any]] = {}
async def create(self, data: dict[str, Any]) -> dict[str, Any]:
record = {
"id": data["id"],
"user_id": data["user_id"],
"type": data["type"],
"title": data.get("title") or "",
"body": data.get("body") or "",
"payload": data.get("payload") or {},
"read": False,
"created_at": _now_iso(),
}
self._items[record["id"]] = record
return copy.deepcopy(record)
async def list_for_user(
self,
user_id: str,
*,
unread_only: bool = False,
limit: int = 50,
) -> list[dict[str, Any]]:
filtered = [r for r in self._items.values() if r["user_id"] == user_id]
if unread_only:
filtered = [r for r in filtered if not r["read"]]
filtered.sort(key=lambda r: r["created_at"], reverse=True)
return [copy.deepcopy(r) for r in filtered[:limit]]
async def unread_count(self, user_id: str) -> int:
return sum(1 for r in self._items.values() if r["user_id"] == user_id and not r["read"])
async def mark_read(self, notification_id: str, user_id: str) -> bool:
record = self._items.get(notification_id)
if record is None or record["user_id"] != user_id:
return False
record["read"] = True
return True
async def mark_all_read(self, user_id: str) -> int:
updated = 0
for record in self._items.values():
if record["user_id"] == user_id and not record["read"]:
record["read"] = True
updated += 1
return updated
async def delete(self, notification_id: str, user_id: str) -> bool:
record = self._items.get(notification_id)
if record is None or record["user_id"] != user_id:
return False
del self._items[notification_id]
return True
@@ -0,0 +1,24 @@
"""ORM model for in-app notifications."""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import JSON, Boolean, DateTime, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
class NotificationRow(Base):
__tablename__ = "notifications"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
type: Mapped[str] = mapped_column(String(64), nullable=False)
title: Mapped[str] = mapped_column(String(256), nullable=False, default="")
body: Mapped[str] = mapped_column(Text, nullable=False, default="")
payload: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False, default=dict)
read: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), index=True)
@@ -0,0 +1,72 @@
"""Helper for writing notifications without callers having to thread a store.
The harness can NOT import ``app.*`` (firewall), so the Gateway app injects
its ``NotificationStore`` instance via ``set_notification_store(...)`` at
startup. Until that happens, every call is a no-op (silently logged) so
non-Gateway entry points (tests, embedded client) don't crash on missing
infrastructure.
"""
from __future__ import annotations
import logging
import uuid
from typing import Any
from deerflow.persistence.notifications.base import NotificationStore
logger = logging.getLogger(__name__)
_store: NotificationStore | None = None
def set_notification_store(store: NotificationStore | None) -> None:
"""Inject the active ``NotificationStore`` instance for the process.
Pass ``None`` to clear (e.g. on shutdown or in tests).
"""
global _store
_store = store
def get_notification_store() -> NotificationStore | None:
return _store
async def notify(
user_id: str,
*,
type: str,
title: str = "",
body: str = "",
payload: dict[str, Any] | None = None,
) -> None:
"""Write a notification for ``user_id``. Never raises — failures are logged.
Args:
user_id: Recipient.
type: Stable identifier (e.g. ``agent_skill_removed``, ``skill_taken_down``).
title: One-line title shown in the inbox.
body: Longer description shown when the user expands the entry.
payload: Structured context (skill_name, agent_id, actor_user_id, ...).
"""
try:
if _store is None:
logger.warning(
"notify(user_id=%s, type=%s) dropped — NotificationStore not configured",
user_id,
type,
)
return
await _store.create(
{
"id": str(uuid.uuid4()),
"user_id": user_id,
"type": type,
"title": title,
"body": body,
"payload": payload or {},
}
)
except Exception:
logger.exception("Failed to write notification (user_id=%s, type=%s)", user_id, type)
@@ -0,0 +1,97 @@
"""SQLAlchemy-backed notification repository."""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import func, select, update
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.notifications.base import NotificationStore
from deerflow.persistence.notifications.model import NotificationRow
def _row_to_dict(row: NotificationRow) -> dict[str, Any]:
data = row.to_dict()
value = data.get("created_at")
if isinstance(value, datetime):
data["created_at"] = value.isoformat()
return data
class NotificationRepository(NotificationStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def create(self, data: dict[str, Any]) -> dict[str, Any]:
row = NotificationRow(
id=data["id"],
user_id=data["user_id"],
type=data["type"],
title=data.get("title") or "",
body=data.get("body") or "",
payload=data.get("payload") or {},
read=False,
created_at=datetime.now(UTC),
)
async with self._sf() as session:
session.add(row)
await session.commit()
await session.refresh(row)
return _row_to_dict(row)
async def list_for_user(
self,
user_id: str,
*,
unread_only: bool = False,
limit: int = 50,
) -> list[dict[str, Any]]:
stmt = select(NotificationRow).where(NotificationRow.user_id == user_id)
if unread_only:
stmt = stmt.where(NotificationRow.read.is_(False))
stmt = stmt.order_by(NotificationRow.created_at.desc()).limit(limit)
async with self._sf() as session:
result = await session.execute(stmt)
return [_row_to_dict(row) for row in result.scalars()]
async def unread_count(self, user_id: str) -> int:
stmt = (
select(func.count())
.select_from(NotificationRow)
.where(NotificationRow.user_id == user_id, NotificationRow.read.is_(False))
)
async with self._sf() as session:
result = await session.execute(stmt)
return int(result.scalar_one() or 0)
async def mark_read(self, notification_id: str, user_id: str) -> bool:
async with self._sf() as session:
row = await session.get(NotificationRow, notification_id)
if row is None or row.user_id != user_id:
return False
if not row.read:
row.read = True
await session.commit()
return True
async def mark_all_read(self, user_id: str) -> int:
stmt = (
update(NotificationRow)
.where(NotificationRow.user_id == user_id, NotificationRow.read.is_(False))
.values(read=True)
)
async with self._sf() as session:
result = await session.execute(stmt)
await session.commit()
return int(result.rowcount or 0)
async def delete(self, notification_id: str, user_id: str) -> bool:
async with self._sf() as session:
row = await session.get(NotificationRow, notification_id)
if row is None or row.user_id != user_id:
return False
await session.delete(row)
await session.commit()
return True
@@ -0,0 +1,13 @@
"""Persistence layer for custom skills (per-user ownership + publish state)."""
from deerflow.persistence.skills.base import SkillStore
from deerflow.persistence.skills.memory import MemorySkillStore
from deerflow.persistence.skills.sql import SkillRepository
__all__ = ["SkillStore", "MemorySkillStore", "SkillRepository", "make_skill_store"]
def make_skill_store(session_factory):
if session_factory is None:
return MemorySkillStore()
return SkillRepository(session_factory)
@@ -0,0 +1,69 @@
"""Abstract persistence API for custom skill ownership + publish state."""
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any
class SkillStore(ABC):
@abstractmethod
async def list_visible(self, user_id: str) -> list[dict[str, Any]]:
"""List all skill records visible to ``user_id``.
Visible = owner, OR published by anyone, OR built-in (owner_user_id IS NULL).
"""
raise NotImplementedError
@abstractmethod
async def get_visible(self, name: str, user_id: str) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def get_any(self, name: str) -> dict[str, Any] | None:
"""Lookup by name regardless of ownership/visibility (admin / cascade use)."""
raise NotImplementedError
@abstractmethod
async def get_owned(self, name: str, user_id: str) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def create(self, data: dict[str, Any]) -> dict[str, Any]:
"""Create a skill record. Caller is responsible for global-name uniqueness check."""
raise NotImplementedError
@abstractmethod
async def ensure_legacy(self, data: dict[str, Any]) -> dict[str, Any]:
"""Idempotently insert a legacy on-disk skill into the DB. Updates the row if it
already exists with ``owner_user_id IS NULL`` (so a re-run can migrate ownership).
"""
raise NotImplementedError
@abstractmethod
async def update(self, name: str, user_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
"""Owner-only update. Returns None if the caller doesn't own the skill."""
raise NotImplementedError
@abstractmethod
async def update_admin(self, name: str, data: dict[str, Any]) -> dict[str, Any] | None:
"""Admin override — caller is responsible for auth check.
Unlike ``update_builtin`` for agents, this accepts ANY skill including
user-owned ones, since the spec lets admin take down others' skills.
"""
raise NotImplementedError
@abstractmethod
async def delete(self, name: str, user_id: str) -> bool:
raise NotImplementedError
@abstractmethod
async def delete_admin(self, name: str) -> bool:
"""Admin-side delete; bypasses ownership check (caller authorizes)."""
raise NotImplementedError
@abstractmethod
async def count_owned(self, user_id: str) -> int:
"""Return the number of skills owned by ``user_id`` (owner_user_id IS NOT NULL)."""
raise NotImplementedError
@@ -0,0 +1,129 @@
"""In-memory ``SkillStore`` implementation for tests."""
from __future__ import annotations
import copy
from datetime import UTC, datetime
from typing import Any
from deerflow.persistence.skills.base import SkillStore
def _now() -> str:
return datetime.now(UTC).isoformat()
def _copy(record: dict[str, Any] | None) -> dict[str, Any] | None:
return copy.deepcopy(record) if record is not None else None
def _with_derived(record: dict[str, Any]) -> dict[str, Any]:
out = dict(record)
out["builtin"] = out.get("owner_user_id") is None
return out
class MemorySkillStore(SkillStore):
def __init__(self) -> None:
self._skills: dict[str, dict[str, Any]] = {}
@staticmethod
def _is_visible(record: dict[str, Any], user_id: str) -> bool:
owner = record.get("owner_user_id")
if owner is None:
return True
if owner == user_id:
return True
return bool(record.get("published"))
async def list_visible(self, user_id: str) -> list[dict[str, Any]]:
return [
_with_derived(_copy(record)) # type: ignore[arg-type]
for record in self._skills.values()
if self._is_visible(record, user_id)
]
async def get_visible(self, name: str, user_id: str) -> dict[str, Any] | None:
record = self._skills.get(name)
if record is None or not self._is_visible(record, user_id):
return None
return _with_derived(_copy(record)) # type: ignore[arg-type]
async def get_any(self, name: str) -> dict[str, Any] | None:
record = self._skills.get(name)
if record is None:
return None
return _with_derived(_copy(record)) # type: ignore[arg-type]
async def get_owned(self, name: str, user_id: str) -> dict[str, Any] | None:
record = self._skills.get(name)
if record is None or record.get("owner_user_id") != user_id:
return None
return _with_derived(_copy(record)) # type: ignore[arg-type]
async def create(self, data: dict[str, Any]) -> dict[str, Any]:
name = data["name"]
if name in self._skills:
raise ValueError(f"Skill '{name}' already exists")
record = {
"name": name,
"owner_user_id": data.get("owner_user_id"),
"published": bool(data.get("published", False)),
"created_at": _now(),
"updated_at": _now(),
}
self._skills[name] = record
return _with_derived(_copy(record)) # type: ignore[arg-type]
async def ensure_legacy(self, data: dict[str, Any]) -> dict[str, Any]:
name = data["name"]
record = self._skills.get(name)
if record is None:
record = {
"name": name,
"owner_user_id": data.get("owner_user_id"),
"published": bool(data.get("published", True)),
"created_at": _now(),
"updated_at": _now(),
}
self._skills[name] = record
elif record.get("owner_user_id") is None and data.get("owner_user_id") is not None:
record["owner_user_id"] = data["owner_user_id"]
record["updated_at"] = _now()
return _with_derived(_copy(record)) # type: ignore[arg-type]
async def update(self, name: str, user_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
record = self._skills.get(name)
if record is None or record.get("owner_user_id") != user_id:
return None
if "published" in data:
record["published"] = bool(data["published"])
record["updated_at"] = _now()
return _with_derived(_copy(record)) # type: ignore[arg-type]
async def update_admin(self, name: str, data: dict[str, Any]) -> dict[str, Any] | None:
record = self._skills.get(name)
if record is None:
return None
if "published" in data:
record["published"] = bool(data["published"])
if "owner_user_id" in data:
record["owner_user_id"] = data["owner_user_id"]
record["updated_at"] = _now()
return _with_derived(_copy(record)) # type: ignore[arg-type]
async def delete(self, name: str, user_id: str) -> bool:
record = self._skills.get(name)
if record is None or record.get("owner_user_id") != user_id:
return False
del self._skills[name]
return True
async def delete_admin(self, name: str) -> bool:
if name not in self._skills:
return False
del self._skills[name]
return True
async def count_owned(self, user_id: str) -> int:
return sum(1 for s in self._skills.values() if s.get("owner_user_id") == user_id)
@@ -0,0 +1,28 @@
"""ORM model for custom skill ownership + publish state.
Note: skill content lives on disk under ``skills/custom/{name}/``; this table
only stores the metadata (owner, published flag, timestamps). The skill
``name`` doubles as the primary key because the filesystem path is already
keyed by name, and we require global name uniqueness across all owners.
"""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, DateTime, String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
class SkillRow(Base):
__tablename__ = "skills"
# Global unique name — must match the on-disk folder name under skills/custom/.
# Hyphen-case or snake_case allowed; see deerflow.skills.validation.
name: Mapped[str] = mapped_column(String(64), primary_key=True)
owner_user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
published: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
@@ -0,0 +1,144 @@
"""SQLAlchemy-backed custom skill repository."""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import func, or_, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.skills.base import SkillStore
from deerflow.persistence.skills.model import SkillRow
def _row_to_dict(row: SkillRow) -> dict[str, Any]:
data = row.to_dict()
data["builtin"] = data.get("owner_user_id") is None
for key in ("created_at", "updated_at"):
value = data.get(key)
if isinstance(value, datetime):
data[key] = value.isoformat()
return data
class SkillRepository(SkillStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
@staticmethod
def _visible_filter(user_id: str):
return or_(
SkillRow.owner_user_id.is_(None),
SkillRow.owner_user_id == user_id,
SkillRow.published.is_(True),
)
async def list_visible(self, user_id: str) -> list[dict[str, Any]]:
stmt = select(SkillRow).where(self._visible_filter(user_id)).order_by(SkillRow.created_at.asc())
async with self._sf() as session:
result = await session.execute(stmt)
return [_row_to_dict(row) for row in result.scalars()]
async def get_visible(self, name: str, user_id: str) -> dict[str, Any] | None:
stmt = select(SkillRow).where(SkillRow.name == name, self._visible_filter(user_id))
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
return _row_to_dict(row) if row is not None else None
async def get_any(self, name: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(SkillRow, name)
return _row_to_dict(row) if row is not None else None
async def get_owned(self, name: str, user_id: str) -> dict[str, Any] | None:
stmt = select(SkillRow).where(SkillRow.name == name, SkillRow.owner_user_id == user_id)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
return _row_to_dict(row) if row is not None else None
async def create(self, data: dict[str, Any]) -> dict[str, Any]:
now = datetime.now(UTC)
row = SkillRow(
name=data["name"],
owner_user_id=data.get("owner_user_id"),
published=bool(data.get("published", False)),
created_at=now,
updated_at=now,
)
async with self._sf() as session:
session.add(row)
await session.commit()
await session.refresh(row)
return _row_to_dict(row)
async def ensure_legacy(self, data: dict[str, Any]) -> dict[str, Any]:
async with self._sf() as session:
row = await session.get(SkillRow, data["name"])
if row is None:
now = datetime.now(UTC)
row = SkillRow(
name=data["name"],
owner_user_id=data.get("owner_user_id"),
published=bool(data.get("published", True)),
created_at=now,
updated_at=now,
)
session.add(row)
elif row.owner_user_id is None and data.get("owner_user_id") is not None:
# Only patch ownership if it was previously null — never overwrite
# an existing owner record on a re-migration.
row.owner_user_id = data["owner_user_id"]
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
return _row_to_dict(row)
async def update(self, name: str, user_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(SkillRow, name)
if row is None or row.owner_user_id != user_id:
return None
if "published" in data:
row.published = bool(data["published"])
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
return _row_to_dict(row)
async def update_admin(self, name: str, data: dict[str, Any]) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(SkillRow, name)
if row is None:
return None
if "published" in data:
row.published = bool(data["published"])
if "owner_user_id" in data:
row.owner_user_id = data["owner_user_id"]
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
return _row_to_dict(row)
async def delete(self, name: str, user_id: str) -> bool:
async with self._sf() as session:
row = await session.get(SkillRow, name)
if row is None or row.owner_user_id != user_id:
return False
await session.delete(row)
await session.commit()
return True
async def delete_admin(self, name: str) -> bool:
async with self._sf() as session:
row = await session.get(SkillRow, name)
if row is None:
return False
await session.delete(row)
await session.commit()
return True
async def count_owned(self, user_id: str) -> int:
stmt = select(func.count()).select_from(SkillRow).where(SkillRow.owner_user_id == user_id)
async with self._sf() as session:
return await session.scalar(stmt) or 0
@@ -0,0 +1,151 @@
"""Cascade handling when a published skill is removed from circulation.
Called from the Gateway skills router whenever a published skill is
unpublished, deleted, or admin-taken-down. Walks every agent on disk that
referenced the skill, strips the skill from its ``config.yaml`` ``skills``
list, and emits an in-app notification to that agent's owner.
The harness/app firewall means we cannot import the FastAPI app side; the
``AgentStore`` instance must be passed in by the caller (the router gets
it from ``app.gateway.deps.get_agent_store``).
"""
from __future__ import annotations
import logging
from pathlib import Path
from typing import Any, Literal
import yaml
from deerflow.config.agents_config import load_agent_config
from deerflow.config.paths import get_paths
from deerflow.persistence.notifications import notify
logger = logging.getLogger(__name__)
CascadeOperation = Literal["unpublish", "delete", "admin_takedown"]
def _list_agent_dirs() -> list[Path]:
agents_dir = get_paths().agents_dir
if not agents_dir.exists():
return []
return [entry for entry in sorted(agents_dir.iterdir()) if entry.is_dir()]
def _remove_skill_from_yaml(config_file: Path, skill_name: str) -> bool:
"""Rewrite ``config.yaml`` removing ``skill_name`` from its ``skills`` list.
Returns True if the file was modified, False otherwise.
"""
if not config_file.exists():
return False
try:
with config_file.open("r", encoding="utf-8") as f:
data: dict[str, Any] = yaml.safe_load(f) or {}
except yaml.YAMLError:
logger.warning("Cascade: failed to parse %s, skipping", config_file)
return False
skills = data.get("skills")
if not isinstance(skills, list) or skill_name not in skills:
return False
data["skills"] = [s for s in skills if s != skill_name]
with config_file.open("w", encoding="utf-8") as f:
yaml.safe_dump(data, f, default_flow_style=False, allow_unicode=True)
return True
_OP_LABELS: dict[CascadeOperation, str] = {
"unpublish": "取消发布",
"delete": "删除",
"admin_takedown": "管理员强制下架",
}
async def cascade_skill_unpublished_or_deleted(
skill_name: str,
*,
actor_user_id: str,
operation: CascadeOperation,
skill_owner_user_id: str | None,
agent_store: Any,
) -> dict[str, Any]:
"""Detach ``skill_name`` from every referencing agent and notify owners.
Self-references (agent.user_id == actor_user_id) are still detached but
the owner gets no in-app notification — the user just did the action,
they know.
On ``admin_takedown`` we additionally notify ``skill_owner_user_id``.
Returns counters for testability: ``{checked, modified, notified}``.
"""
counters = {"checked": 0, "modified": 0, "notified": 0}
label = _OP_LABELS.get(operation, operation)
for agent_dir in _list_agent_dirs():
config_file = agent_dir / "config.yaml"
if not config_file.exists():
continue
counters["checked"] += 1
# Quick read to decide whether we need to mutate at all.
try:
agent_cfg = load_agent_config(agent_dir.name)
except Exception:
logger.debug("Cascade: failed to load %s, skipping", agent_dir.name, exc_info=True)
continue
if agent_cfg is None or not agent_cfg.skills or skill_name not in agent_cfg.skills:
continue
modified = _remove_skill_from_yaml(config_file, skill_name)
if not modified:
continue
counters["modified"] += 1
logger.info(
"Cascade: removed skill '%s' from agent '%s/config.yaml' (op=%s)",
skill_name,
agent_dir.name,
operation,
)
# Look up agent owner from DB to decide where to send the notification.
record = await agent_store.get_any(agent_dir.name)
agent_owner = record.get("user_id") if record else None
if agent_owner and agent_owner != actor_user_id:
agent_display = (record.get("name") if record else None) or agent_dir.name
await notify(
agent_owner,
type="agent_skill_removed",
title=f"专家 “{agent_display}” 的一个技能已失效",
body=(
f"技能 “{skill_name}” 被 {label},已从该专家的技能列表中自动移除。"
),
payload={
"skill_name": skill_name,
"agent_id": agent_dir.name,
"agent_name": agent_display,
"operation": operation,
"actor_user_id": actor_user_id,
},
)
counters["notified"] += 1
if operation == "admin_takedown" and skill_owner_user_id and skill_owner_user_id != actor_user_id:
await notify(
skill_owner_user_id,
type="skill_taken_down",
title=f"你的技能 “{skill_name}” 已被管理员下架",
body="管理员对该技能执行了强制下架。如有疑问请联系管理员。",
payload={
"skill_name": skill_name,
"operation": operation,
"actor_user_id": actor_user_id,
},
)
counters["notified"] += 1
return counters
@@ -0,0 +1,835 @@
"""Skill curator — automatic lifecycle transitions and LLM consolidation pass.
Per-user since the multitenancy upgrade: every public function takes a
``user_id`` argument and the on-disk state lives under
``{base_dir}/users/{user_id}/.curator_state`` (alongside the per-user
memory.json and threads/ dirs).
"""
from __future__ import annotations
import asyncio
import concurrent.futures
import json
import logging
import os
import re
import tempfile
import threading
from collections.abc import Coroutine
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any, TypeVar
logger = logging.getLogger(__name__)
_CURATOR_STATE_FILENAME = ".curator_state"
_T = TypeVar("_T")
def _run_async_blocking(coro: Coroutine[Any, Any, _T]) -> _T:
"""Run an async coroutine to completion from sync curator code.
Curator is invoked from a background thread, but ``agent.invoke()`` inside
``_run_llm_review`` (LangChain 1.x ``create_agent``) spins up its own
asyncio loop which can leave the calling thread in an "already has a
running loop" state. In that case raw ``asyncio.run(coro)`` raises
``RuntimeError`` *after* the coroutine object has been materialised but
before anyone awaits it — producing the ``coroutine was never awaited``
RuntimeWarning that pollutes our logs.
This helper detects a running loop in the caller's thread and offloads
to a dedicated worker thread; otherwise it falls back to ``asyncio.run``.
Either way it guarantees the coroutine is closed on the failure path,
so no RuntimeWarning ever escapes.
"""
try:
asyncio.get_running_loop()
in_loop = True
except RuntimeError:
in_loop = False
if in_loop:
try:
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
return pool.submit(asyncio.run, coro).result()
except BaseException:
# ``asyncio.run`` ran in the worker (it consumes the coroutine on
# success or surfaces the inner exception); but if the executor
# itself failed to submit, close to be safe.
try:
coro.close()
except Exception:
pass
raise
try:
return asyncio.run(coro)
except BaseException:
try:
coro.close()
except Exception:
pass
raise
def _get_skills_root() -> Path:
from deerflow.skills.storage import get_or_new_skill_storage
return Path(get_or_new_skill_storage().get_skills_root_path())
def _curator_state_path(user_id: str | None) -> Path:
"""Return the state file path. Per-user when user_id is known; falls back
to the global ``skills/.curator_state`` for legacy callers."""
if not user_id:
return _get_skills_root() / _CURATOR_STATE_FILENAME
from deerflow.config.paths import get_paths
return get_paths().user_dir(user_id) / _CURATOR_STATE_FILENAME
def _read_curator_state(user_id: str | None) -> dict:
path = _curator_state_path(user_id)
if not path.exists():
return {"run_count": 0, "paused": False}
try:
return json.loads(path.read_text(encoding="utf-8"))
except Exception:
return {"run_count": 0, "paused": False}
def _write_curator_state(state: dict, user_id: str | None) -> None:
path = _curator_state_path(user_id)
try:
path.parent.mkdir(parents=True, exist_ok=True)
tmp_fd, tmp_path = tempfile.mkstemp(dir=path.parent, suffix=".tmp")
try:
with os.fdopen(tmp_fd, "w", encoding="utf-8") as f:
json.dump(state, f, indent=2, default=str)
os.replace(tmp_path, path)
except Exception:
try:
os.unlink(tmp_path)
except Exception:
pass
raise
except Exception:
logger.debug("Failed to write curator state", exc_info=True)
def _get_curator_config(user_id: str | None = None):
"""Return the curator config object for ``user_id``.
Reads ``config.yaml.skill_evolution.curator`` as the baseline and merges
the per-user override file (``users/{user_id}/skill_evolution.json``) on
top. With ``user_id=None`` falls back to the global default.
"""
try:
from deerflow.config.skill_evolution_runtime import load_skill_evolution_config_for_user
skill_evo = load_skill_evolution_config_for_user(user_id)
return getattr(skill_evo, "curator", None)
except Exception:
return None
def _list_user_owned_skill_names(user_id: str) -> set[str] | None:
"""Return skill names owned by ``user_id`` (used to filter transitions).
Returns None if the SkillStore is unreachable — curator then degrades to
"process every on-disk agent-created skill" (legacy behaviour).
"""
try:
from deerflow.persistence.engine import get_session_factory
from deerflow.persistence.skills import make_skill_store
factory = get_session_factory()
if factory is None:
return None
store = make_skill_store(factory)
async def _run() -> list[dict]:
return await store.list_visible(user_id)
records = _run_async_blocking(_run())
return {r["name"] for r in records if r.get("owner_user_id") == user_id}
except Exception:
logger.debug("_list_user_owned_skill_names failed", exc_info=True)
return None
def _collect_referenced_skill_names() -> set[str]:
"""Scan every agent's config.yaml and return the union of their ``skills``
fields.
Used as the "do not touch" set during curator transitions and LLM
candidate selection — any skill currently bound to *any* agent
(including the owner's own) is exempt from automatic stale/archive
and from being suggested for merging.
Returns an empty set on filesystem errors (best-effort, non-fatal).
"""
import yaml
from deerflow.config.paths import get_paths
referenced: set[str] = set()
try:
agents_dir = get_paths().agents_dir
except Exception:
return referenced
if not agents_dir.exists():
return referenced
for entry in agents_dir.iterdir():
if not entry.is_dir():
continue
config_file = entry / "config.yaml"
if not config_file.exists():
continue
try:
with config_file.open("r", encoding="utf-8") as f:
data = yaml.safe_load(f) or {}
except Exception:
logger.debug("_collect_referenced_skill_names: failed to parse %s", config_file, exc_info=True)
continue
skills = data.get("skills")
if isinstance(skills, list):
for s in skills:
if isinstance(s, str):
referenced.add(s)
return referenced
def _is_skill_published(skill_name: str) -> bool:
"""Return True if ``skill_name`` is marked ``published=True`` in SkillStore.
Best-effort — returns False on any error so a transient DB hiccup never
causes a published skill to be wrongly archived (the caller treats
"published=True" as exempt, so False = "fail open to processing", which
only puts a skill *back* into curator scope when DB is unavailable; in
that degraded mode curator legacy behaviour applies).
"""
try:
from deerflow.persistence.engine import get_session_factory
from deerflow.persistence.skills import make_skill_store
factory = get_session_factory()
if factory is None:
return False
store = make_skill_store(factory)
async def _run() -> dict | None:
return await store.get_any(skill_name)
record = _run_async_blocking(_run())
return bool(record and record.get("published", False))
except Exception:
logger.debug("_is_skill_published(%s) failed", skill_name, exc_info=True)
return False
def _eligible_for_curation(
name: str,
entry: dict,
*,
owned_set: set[str] | None,
referenced_set: set[str],
) -> bool:
"""Three-condition gate for whether curator may touch a skill.
Curator only acts on skills that ALL of:
1. source == "agent" (caller is expected to have already filtered, but
we double-check here for safety)
2. NOT referenced by any agent's skills config
3. NOT published (queried via SkillStore)
``owned_set`` is the set of skill names owned by the running user. When
None, the legacy "process every agent-created skill" fallback applies
for ownership only (the other two conditions still hold).
"""
if entry.get("source") != "agent":
return False
if owned_set is not None and name not in owned_set:
return False
if name in referenced_set:
return False
if _is_skill_published(name):
return False
return True
def should_run_now(user_id: str) -> bool:
"""Return True if the curator interval has elapsed for ``user_id`` and it
is not paused."""
curator_cfg = _get_curator_config()
if curator_cfg is None or not getattr(curator_cfg, "enabled", True):
return False
state = _read_curator_state(user_id)
if state.get("paused"):
return False
last_run = state.get("last_run_at")
if not last_run:
return True
interval_hours = getattr(curator_cfg, "interval_hours", 168)
try:
last_dt = datetime.fromisoformat(last_run)
return datetime.now(UTC) >= last_dt + timedelta(hours=interval_hours)
except Exception:
return True
def apply_automatic_transitions(user_id: str, now: datetime | None = None) -> dict:
"""Pure age-based state transitions — no LLM involved.
Only processes agent-created skills that are ALL of:
- owned by ``user_id``
- not referenced by any agent's ``skills`` config
- not currently published
Skips pinned skills. Returns counters: {marked_stale, archived, checked,
skipped_referenced, skipped_published}.
"""
from deerflow.skills.usage import STATE_ACTIVE, STATE_ARCHIVED, STATE_STALE, agent_created_entries, latest_activity_at, set_state
if now is None:
now = datetime.now(UTC)
curator_cfg = _get_curator_config()
stale_days = getattr(curator_cfg, "stale_after_days", 30) if curator_cfg else 30
archive_days = getattr(curator_cfg, "archive_after_days", 90) if curator_cfg else 90
owned = _list_user_owned_skill_names(user_id)
referenced = _collect_referenced_skill_names()
raw_entries = agent_created_entries()
skipped_referenced = 0
skipped_published = 0
eligible: dict[str, dict] = {}
for name, entry in raw_entries.items():
if owned is not None and name not in owned:
continue
if name in referenced:
skipped_referenced += 1
continue
if _is_skill_published(name):
skipped_published += 1
continue
eligible[name] = entry
counters = {
"marked_stale": 0,
"archived": 0,
"checked": len(eligible),
"skipped_referenced": skipped_referenced,
"skipped_published": skipped_published,
}
from deerflow.skills import curator_log
for name, entry in eligible.items():
if entry.get("pinned"):
continue
last_activity_str = latest_activity_at(name)
if not last_activity_str:
continue
try:
last_dt = datetime.fromisoformat(last_activity_str)
if last_dt.tzinfo is None:
last_dt = last_dt.replace(tzinfo=UTC)
except Exception:
continue
days_idle = (now - last_dt).days
current_state = entry.get("state", STATE_ACTIVE)
if days_idle >= archive_days and current_state != STATE_ARCHIVED:
set_state(name, STATE_ARCHIVED)
counters["archived"] += 1
curator_log.record(
"info",
f" 自动归档 [{name}]:已闲置 {days_idle} 天(阈值 {archive_days} 天)",
user_id=user_id,
)
elif days_idle >= stale_days and current_state == STATE_ACTIVE:
set_state(name, STATE_STALE)
counters["marked_stale"] += 1
curator_log.record(
"info",
f" 标记过时 [{name}]:已闲置 {days_idle} 天(阈值 {stale_days} 天)",
user_id=user_id,
)
return counters
def _render_candidate_list(user_id: str) -> str:
"""List of skill candidates handed to the LLM for merge/archive review.
Same three-condition gate as ``apply_automatic_transitions``: only
agent-created, not-referenced-by-any-agent, not-published skills owned
by ``user_id`` show up here. The LLM literally cannot see (and thus
cannot suggest action on) anything outside that set.
"""
from deerflow.skills.usage import STATE_ARCHIVED, all_entries
owned = _list_user_owned_skill_names(user_id)
referenced = _collect_referenced_skill_names()
entries: dict[str, dict] = {}
for k, v in all_entries().items():
if not _eligible_for_curation(k, v, owned_set=owned, referenced_set=referenced):
continue
if v.get("state") == STATE_ARCHIVED:
continue
entries[k] = v
if not entries:
return "No agent-created active skills found."
lines = []
for name, entry in sorted(entries.items()):
state = entry.get("state", "active")
use = entry.get("use_count", 0)
view = entry.get("view_count", 0)
patch = entry.get("patch_count", 0)
last = entry.get("last_used_at") or entry.get("created_at") or "unknown"
lines.append(f"- {name} [state={state}, use={use}, view={view}, patches={patch}, last_active={last}]")
return "\n".join(lines)
_CURATOR_REVIEW_PROMPT = (
"You are running as DeerFlow's background skill CURATOR. This is an "
"UMBRELLA-BUILDING consolidation pass, not a passive audit and not a "
"duplicate-finder.\n\n"
"The goal of the skill collection is a LIBRARY OF CLASS-LEVEL "
"INSTRUCTIONS AND EXPERIENTIAL KNOWLEDGE. A collection of hundreds of "
"narrow skills where each one captures one session's specific bug is "
"a FAILURE of the library — not a feature. An agent searching skills "
"matches on descriptions, not on exact names; one broad umbrella "
"skill with labeled subsections beats five narrow siblings for "
"discoverability, not the other way around.\n\n"
"The right target shape is CLASS-LEVEL skills with rich SKILL.md "
"bodies + `references/`, `templates/`, and `scripts/` subfiles for "
"session-specific detail — not one-session-one-skill micro-entries.\n\n"
"Hard rules — do not violate:\n"
"1. DO NOT touch public (built-in) skills. The candidate list below "
"is already filtered to agent-created custom skills only.\n"
"2. DO NOT delete any skill. Archiving (a soft state flip via the "
"skill_archive tool — the file stays in custom/<name>/ and the user "
"can restore it from the Web UI) is the maximum action you may take. "
"Archives are recoverable; deletion is not. "
"When you call skill_archive(name, reason) you MUST pass a `reason` "
"string written **in Chinese**, in plain language a human operator "
"would read — NOT English jargon or terse debug notes. Examples of "
"acceptable reasons: '已合并到 code-formatter,该技能同时覆盖 Python 与 JS', "
"'与 foo-bar 内容重复,保留 foo-bar', '重构后过时,被 api-v2-search 替代', "
"'一次性 POC 实验,无复用价值'. Bad reasons (do NOT use): 'merged', "
"'duplicate', 'obsolete', 'consolidated'. The reason is shown in the "
"Web UI run-history for user audit, so write it as you would explain "
"it to the user themselves.\n"
"3. DO NOT touch skills shown as pinned=yes. Skip them entirely.\n"
"4. DO NOT use usage counters as a reason to skip consolidation. The "
"counters are new and often mostly zero. Judge overlap on CONTENT, "
"not on use_count. 'use=0' is not evidence a skill is valuable; it's "
"absence of evidence either way.\n"
"5. DO NOT reject consolidation on the grounds that 'each skill has "
"a distinct trigger'. Pairwise distinctness is the wrong bar. The "
"right bar is: 'would a human maintainer write this as N separate "
"skills, or as one skill with N labeled subsections?' When the "
"answer is the latter, merge.\n\n"
"How to work — not optional:\n"
"1. Scan the full candidate list. Identify PREFIX CLUSTERS (skills "
"sharing a first word or domain keyword). Examples you are likely "
"to find: search-*, code-*, data-*, api-*, file-*, analysis-*, "
"workflow-*, report-*, monitor-*, debug-*, python-*, security-*, etc. "
"Expect 10-25 clusters.\n"
"2. For each cluster with 2+ members, do NOT ask 'are these pairs "
"overlapping?' — ask 'what is the UMBRELLA CLASS these skills all "
"serve? Would a maintainer name that class and write one skill for "
"it?' If yes, pick (or create) the umbrella and absorb the siblings "
"into it.\n"
"3. Three ways to consolidate — use the right one per cluster:\n"
" a. MERGE INTO EXISTING UMBRELLA — one skill in the cluster is "
"already broad enough to be the umbrella. Patch it to add a labeled "
"section for each sibling's unique insight, then archive the "
"siblings.\n"
" b. CREATE A NEW UMBRELLA SKILL.md — no existing member is broad "
"enough. Create a new class-level skill whose SKILL.md covers the "
"shared workflow and has short labeled subsections. Archive the "
"now-absorbed narrow siblings.\n"
" c. DEMOTE TO REFERENCES/TEMPLATES/SCRIPTS — a sibling has "
"narrow-but-valuable session-specific content. Move it into the "
"umbrella's appropriate support directory:\n"
" • `references/<topic>.md` for session-specific detail OR "
"condensed knowledge banks (quoted research, API docs excerpts, "
"domain notes, provider quirks, reproduction recipes)\n"
" • `templates/<name>.<ext>` for starter files meant to be "
"copied and modified\n"
" • `scripts/<name>.<ext>` for statically re-runnable actions "
"(verification scripts, fixture generators, probes)\n"
" Then archive the old sibling.\n"
"4. Also flag skills whose NAME is too narrow (contains a specific "
"issue number, a feature codename, a specific error string, an "
"'audit' / 'diagnosis' / 'salvage' session artifact). These almost "
"always belong as a subsection or support file under a class-level "
"umbrella.\n"
"5. Iterate. After one consolidation round, scan the remaining set "
"and look for the NEXT umbrella opportunity. Don't stop after 3 "
"merges.\n\n"
"'keep' is a legitimate decision ONLY when the skill is already a "
"class-level umbrella and none of the proposed merges would improve "
"discoverability. 'This is narrow but distinct from its siblings' "
"is NOT a reason to keep — it's a reason to move it under an "
"umbrella as a subsection or support file.\n\n"
"Expected output: real umbrella-ification. Process every obvious "
"cluster. If you end the pass with fewer than 10 archives, you "
"stopped too early — go back and look at the clusters you left "
"alone.\n\n"
"When done, write a human summary AND a structured machine-readable "
"block so downstream tooling can distinguish consolidation from "
"pruning. Format EXACTLY:\n\n"
"## Structured summary (required)\n"
"```yaml\n"
"consolidations:\n"
" - from: <old-skill-name>\n"
" into: <umbrella-skill-name>\n"
" reason: <one short sentence — why merged, not just 'similar'>\n"
"prunings:\n"
" - name: <skill-name>\n"
" reason: <one short sentence — why archived with no merge target>\n"
"```\n\n"
"Every skill you archived via skill_archive MUST appear in exactly one "
"of the two lists. If you consolidated X into umbrella Y (patched Y, wrote "
"a references file to Y, or created Y with X's content absorbed), X "
"goes under `consolidations` with `into: Y`. If you archived X with "
"no absorption — truly stale, irrelevant, or obsolete — X goes under "
"`prunings`. Leave a list empty (`consolidations: []`) if none. Do "
"not omit the block. The block comes AFTER your human-readable "
"summary of clusters processed, patches made, and decisions left alone."
)
def _make_skill_archive_tool(user_id: str):
"""Curator-only tool: soft-archive a skill (sets state, files stay put)."""
from langchain.tools import tool as lc_tool
from deerflow.skills import curator_log
from deerflow.skills.storage import get_or_new_skill_storage
@lc_tool("skill_archive")
def _skill_archive(name: str, reason: str) -> str:
"""Soft-archive an agent-created skill (sets state=archived; files stay in custom/).
The user can recover the skill from the Web UI's archive drawer.
This is intentionally non-destructive — the on-disk SKILL.md and any
supporting files are preserved so nothing is lost.
Args:
name: The skill name to archive.
reason: 必填。用**中文**写一句话,清晰说明为什么归档这个技能 —
直接给读它的运维人员看,不要写英文术语 / 调试信息。例如:
- "已合并到 code-formatter,该技能同时覆盖 Python 与 JS"
- "重构后过时,被 api-v2-search 替代"
- "与现有 foo-bar 技能重复,内容完全一致"
- "一次性 POC 实验,无复用价值"
- "命名过窄(含具体 issue 号),已并入通用伞技能 X"
此原因会展示在策展日志和 Web UI 运行历史中,供用户审计。
"""
from deerflow.skills.usage import STATE_ARCHIVED, get_entry, set_state
# Defensive: tolerate empty/None reason but log a clear placeholder
# rather than crashing the curator run.
reason_text = (reason or "").strip() or "未提供原因"
storage = get_or_new_skill_storage()
if not storage.custom_skill_exists(name):
curator_log.record("warning", f"归档:未找到技能 [{name}],或它已被归档", user_id=user_id)
return f"Skill '{name}' not found or already archived."
entry = get_entry(name)
if entry.get("pinned"):
curator_log.record("info", f"归档:技能 [{name}] 已被钉选,跳过", user_id=user_id)
return f"Skill '{name}' is pinned — skipping."
if entry.get("state") == STATE_ARCHIVED:
curator_log.record("info", f"归档:技能 [{name}] 已是归档状态,跳过", user_id=user_id)
return f"Skill '{name}' is already archived."
# Safety net: even if the LLM hallucinated a name not in the
# candidate list, refuse to archive anything that is published or
# currently referenced by an agent.
if _is_skill_published(name):
curator_log.record(
"warning",
f"归档:已拒绝处理 [{name}],该技能已发布(策展不允许操作已发布的技能)",
user_id=user_id,
)
return f"Refused: skill '{name}' is published; manual unpublish is required first."
referenced = _collect_referenced_skill_names()
if name in referenced:
curator_log.record(
"warning",
f"归档:已拒绝处理 [{name}],该技能正被某个 Agent 引用",
user_id=user_id,
)
return f"Refused: skill '{name}' is still referenced by one or more agents."
# Soft archive: flip the lifecycle state. The skill directory stays in
# ``custom/<name>/`` untouched so the user can recover it from the
# Web UI's archive drawer (which lists skills by state, not by location).
try:
set_state(name, STATE_ARCHIVED)
except Exception as e:
curator_log.record("error", f"归档失败:技能 [{name}] — {e}", user_id=user_id)
return f"Archive failed: {e}"
curator_log.record(
"info",
f"已归档(软):[{name}] — 原因:{reason_text} — 文件保留在 custom/{name}/",
user_id=user_id,
)
return f"Archived '{name}' (reason: {reason_text}; files preserved in custom/{name}/)."
return _skill_archive
def _run_llm_review(candidate_list: str, *, user_id: str) -> str:
"""Run the curator consolidation pass as a React agent with skill tools bound."""
from deerflow.skills import curator_log
try:
from langchain.agents import create_agent
from langchain_core.messages import HumanMessage
from deerflow.agents.middlewares.tool_error_handling_middleware import ToolErrorHandlingMiddleware
from deerflow.models import create_chat_model
from deerflow.tools.builtins.skill_tools import skill_list_tool, skill_view_tool
from deerflow.tools.skill_manage_tool import skill_manage_tool
model = create_chat_model()
tools = [skill_list_tool, skill_view_tool, skill_manage_tool, _make_skill_archive_tool(user_id)]
# Reuse the lead-agent error middleware: when the LLM mis-calls a tool
# (wrong action / missing required args), the exception is converted
# into a ToolMessage(status="error") so the LLM can read the error and
# retry, instead of aborting the entire review pass.
# NOTE: ``create_react_agent`` (langgraph.prebuilt) does NOT accept a
# middleware kwarg — use LangChain 1.x's ``create_agent`` which does.
agent = create_agent(model=model, tools=tools, middleware=[ToolErrorHandlingMiddleware()])
curator_log.record("info", "LLM 评审:开始调用 React Agent", user_id=user_id)
prompt = _CURATOR_REVIEW_PROMPT + f"\n\nSkill candidates for this pass:\n{candidate_list}"
result = agent.invoke({"messages": [HumanMessage(content=prompt)]})
messages = result.get("messages", [])
for msg in reversed(messages):
if not getattr(msg, "tool_calls", None):
content = getattr(msg, "content", "")
if isinstance(content, str) and content.strip():
curator_log.record("info", "LLM 评审:Agent 已返回最终结论", user_id=user_id)
return content
if isinstance(content, list):
text = " ".join(
block.get("text", "") if isinstance(block, dict) else str(block)
for block in content
).strip()
curator_log.record("info", "LLM 评审:Agent 已返回最终结论", user_id=user_id)
return text
curator_log.record("warning", "LLM 评审:本次未产出最终回复", user_id=user_id)
return ""
except Exception as e:
logger.warning("Curator LLM review failed: %s", e)
curator_log.record("warning", f"LLM 评审失败:{e}", user_id=user_id)
return ""
def _parse_structured_summary(text: str) -> dict:
pattern = re.compile(r"```yaml\s*(.*?)```", re.DOTALL)
match = pattern.search(text)
if not match:
return {"consolidations": [], "prunings": []}
try:
import yaml
data = yaml.safe_load(match.group(1)) or {}
return {"consolidations": data.get("consolidations") or [], "prunings": data.get("prunings") or []}
except Exception:
return {"consolidations": [], "prunings": []}
def run_curator_review(user_id: str) -> dict:
"""Main curator run for ``user_id``: transitions → LLM consolidation → state.
Only skills owned by ``user_id`` are touched.
"""
from deerflow.skills import curator_log
curator_log.start_new_run(user_id=user_id)
now = datetime.now(UTC)
logger.info("Curator[%s]: starting run at %s", user_id, now.isoformat())
curator_log.record("info", f"本次策展开始,启动时间 {now.isoformat()}", user_id=user_id)
transition_counters = apply_automatic_transitions(user_id, now)
logger.info("Curator[%s]: transitions=%s", user_id, transition_counters)
curator_log.record(
"info",
(
f"按年龄自动流转完成:已检查 {transition_counters['checked']} 个,"
f"标记陈旧 {transition_counters['marked_stale']} 个,"
f"自动归档 {transition_counters['archived']} 个"
),
user_id=user_id,
)
candidate_list = _render_candidate_list(user_id)
candidate_line_count = 0 if not candidate_list.strip() else len([line for line in candidate_list.splitlines() if line.startswith("- ")])
curator_log.record(
"info", f"进入 LLM 评审的候选技能:共 {candidate_line_count} 个", user_id=user_id
)
llm_text = _run_llm_review(candidate_list, user_id=user_id)
suggestions = _parse_structured_summary(llm_text)
curator_log.record(
"info",
(
f"LLM 建议已解析:建议合并 {len(suggestions['consolidations'])} 项,"
f"建议删减 {len(suggestions['prunings'])} 项"
),
user_id=user_id,
)
# Surface per-item reasoning so operators can audit *why* each action
# was taken, not just the count.
for item in suggestions.get("consolidations", []) or []:
if not isinstance(item, dict):
continue
name = item.get("from", "?")
into = item.get("into", "?")
reason = (item.get("reason") or "无说明").strip()
curator_log.record(
"info",
f" 合并建议 [{name}] → [{into}]:{reason}",
user_id=user_id,
)
for item in suggestions.get("prunings", []) or []:
if not isinstance(item, dict):
continue
name = item.get("name", "?")
reason = (item.get("reason") or "无说明").strip()
curator_log.record(
"info",
f" 删减建议 [{name}]:{reason}",
user_id=user_id,
)
state = _read_curator_state(user_id)
state["last_run_at"] = now.isoformat()
state["run_count"] = state.get("run_count", 0) + 1
state["last_run_summary"] = (
f"自动流转:标记陈旧 {transition_counters['marked_stale']} 个、"
f"归档 {transition_counters['archived']} 个;"
f"LLM:建议合并 {len(suggestions['consolidations'])} 项"
)
state["last_consolidation_suggestions"] = suggestions
_write_curator_state(state, user_id)
logger.info("Curator[%s]: run complete — %s", user_id, state["last_run_summary"])
curator_log.record("info", f"本次策展已完成 — {state['last_run_summary']}", user_id=user_id)
return {"transitions": transition_counters, "suggestions": suggestions}
class _CuratorUser:
"""Minimal ``CurrentUser`` protocol implementation for background runs.
Curator threads are spawned via ``threading.Thread`` (not an asyncio task),
so the ``deerflow.runtime.user_context`` ContextVar starts unset. Without
a stand-in, any tool the LLM review invokes (e.g. ``skill_manage(create)``)
would call ``get_effective_user_id()`` and silently fall back to the
``DEFAULT_USER_ID`` sentinel — causing newly-created skills to be owned
by ``"default"`` instead of the user the curator is running for.
"""
__slots__ = ("id",)
def __init__(self, user_id: str) -> None:
self.id = user_id
def _curator_thread_target(user_id: str) -> None:
from deerflow.runtime.user_context import reset_current_user, set_current_user
from deerflow.skills import curator_log
# Bind the per-thread user context for the entire curator run so every
# downstream tool / repository call resolves the correct user_id.
token = set_current_user(_CuratorUser(user_id))
try:
run_curator_review(user_id)
except Exception as exc:
logger.exception("Curator[%s] background run failed", user_id)
curator_log.record("error", f"策展后台任务失败:{exc}", user_id=user_id)
finally:
reset_current_user(token)
async def maybe_run_curator() -> None:
"""Async startup entry — kicks off a thread per user whose interval elapsed.
Skips users with no DB-tracked entries (a fresh deployment) and never
blocks lifespan startup on failure.
"""
try:
from sqlalchemy import select
from deerflow.persistence.engine import get_session_factory
from deerflow.persistence.user.model import UserRow
factory = get_session_factory()
if factory is None:
return
async with factory() as session:
result = await session.execute(select(UserRow.id))
user_ids = [str(row[0]) for row in result.all()]
except Exception:
logger.exception("Curator: failed to enumerate users at startup")
return
for user_id in user_ids:
try:
if not should_run_now(user_id):
continue
logger.info("Curator[%s]: interval elapsed, spawning background thread", user_id)
thread = threading.Thread(
target=_curator_thread_target,
args=(user_id,),
daemon=True,
name=f"skill-curator-{user_id}",
)
thread.start()
except Exception:
logger.exception("Curator: failed to spawn for user %s", user_id)
def get_curator_status(user_id: str) -> dict:
"""Return curator state for ``user_id``."""
from deerflow.skills.usage import STATE_ACTIVE, STATE_ARCHIVED, STATE_STALE, agent_created_entries
state = _read_curator_state(user_id)
owned = _list_user_owned_skill_names(user_id)
entries = agent_created_entries()
if owned is not None:
entries = {name: entry for name, entry in entries.items() if name in owned}
counts: dict[str, int] = {STATE_ACTIVE: 0, STATE_STALE: 0, STATE_ARCHIVED: 0}
for entry in entries.values():
s = entry.get("state", STATE_ACTIVE)
counts[s] = counts.get(s, 0) + 1
cfg = _get_curator_config()
return {
"enabled": cfg is not None and getattr(cfg, "enabled", True),
"run_count": state.get("run_count", 0),
"last_run_at": state.get("last_run_at"),
"last_run_summary": state.get("last_run_summary"),
"paused": state.get("paused", False),
"agent_created_skills": {
"total": len(entries),
"active": counts[STATE_ACTIVE],
"stale": counts[STATE_STALE],
"archived": counts[STATE_ARCHIVED],
},
}
@@ -0,0 +1,111 @@
"""In-memory ring buffer holding the most recent curator run's execution log.
Now sharded per-user — each ``user_id`` gets its own deque so users only see
their own curator activity from the Web UI.
The buffer is intentionally process-local and non-persistent. The Gateway
exposes it over ``GET /api/curator/logs`` with the active user id resolved
from the request context.
All public functions swallow internal exceptions so a logging failure can
never break the curator run itself.
"""
from __future__ import annotations
import logging
import threading
from collections import deque
from datetime import UTC, datetime
from typing import Any
logger = logging.getLogger(__name__)
_MAX_RECORDS = 200
# Reserved bucket key for "no user context" callers (legacy code paths or
# tests). Real curator runs always supply a concrete user_id.
_DEFAULT_BUCKET = "__default__"
_lock = threading.Lock()
_buffers: dict[str, deque[dict[str, Any]]] = {}
def _bucket_key(user_id: str | None) -> str:
return user_id or _DEFAULT_BUCKET
def _get_or_create(user_id: str | None) -> deque[dict[str, Any]]:
key = _bucket_key(user_id)
buffer = _buffers.get(key)
if buffer is None:
buffer = deque(maxlen=_MAX_RECORDS)
_buffers[key] = buffer
return buffer
def _now_iso() -> str:
return datetime.now(UTC).isoformat()
def start_new_run(user_id: str | None = None) -> None:
"""Clear the user's buffer and emit a ``run_started`` record."""
try:
with _lock:
buffer = _get_or_create(user_id)
buffer.clear()
buffer.append({"ts": _now_iso(), "level": "info", "message": "run_started"})
except Exception:
logger.debug("curator_log.start_new_run failed", exc_info=True)
def record(level: str, message: str, *, user_id: str | None = None, **fields: Any) -> None:
"""Append a single log record to the user's buffer. Never raises."""
try:
entry: dict[str, Any] = {
"ts": _now_iso(),
"level": str(level or "info"),
"message": str(message),
}
if fields:
entry.update({k: v for k, v in fields.items() if k not in entry})
with _lock:
buffer = _get_or_create(user_id)
buffer.append(entry)
except Exception:
logger.debug("curator_log.record failed", exc_info=True)
def get_recent(user_id: str | None = None) -> list[dict[str, Any]]:
"""Return a snapshot of records for ``user_id``. Never raises."""
try:
with _lock:
buffer = _buffers.get(_bucket_key(user_id))
if buffer is None:
return []
return list(buffer)
except Exception:
logger.debug("curator_log.get_recent failed", exc_info=True)
return []
def clear(user_id: str | None = None) -> None:
"""Drop a user's records. With ``user_id=None`` drops the default bucket.
Use ``clear_all()`` for cross-user cleanup (tests).
"""
try:
with _lock:
buffer = _buffers.get(_bucket_key(user_id))
if buffer is not None:
buffer.clear()
except Exception:
logger.debug("curator_log.clear failed", exc_info=True)
def clear_all() -> None:
"""Drop every user's records. Used by tests."""
try:
with _lock:
_buffers.clear()
except Exception:
logger.debug("curator_log.clear_all failed", exc_info=True)
@@ -189,6 +189,7 @@ class LocalSkillStorage(SkillStorage):
*,
overwrite: bool = False,
require_skill_suffix: bool = True,
target_category: SkillCategory = SkillCategory.CUSTOM,
) -> dict:
from deerflow.skills.installer import (
SkillAlreadyExistsError,
@@ -199,10 +200,10 @@ class LocalSkillStorage(SkillStorage):
)
from deerflow.skills.validation import _validate_skill_frontmatter
logger.info("Installing skill from %s (overwrite=%s)", archive_path, overwrite)
logger.info("Installing skill from %s (overwrite=%s, target_category=%s)", archive_path, overwrite, target_category)
custom_dir = self._host_root / "custom"
custom_dir.mkdir(parents=True, exist_ok=True)
category_dir = self._host_root / target_category.value
category_dir.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory() as tmp:
tmp_path = Path(tmp)
@@ -218,10 +219,12 @@ class LocalSkillStorage(SkillStorage):
if not skill_name or "/" in skill_name or "\\" in skill_name or ".." in skill_name:
raise ValueError(f"Invalid skill name: {skill_name}")
if self.public_skill_exists(skill_name):
# Custom installs cannot overwrite built-in public skills.
# Public installs (admin-only) are allowed to replace existing public skills when overwrite=True.
if target_category == SkillCategory.CUSTOM and self.public_skill_exists(skill_name):
raise SkillAlreadyExistsError(f"Skill '{skill_name}' conflicts with a built-in public skill and cannot be overwritten")
target = custom_dir / skill_name
target = category_dir / skill_name
replaced = False
if target.exists():
if not overwrite:
@@ -231,7 +234,7 @@ class LocalSkillStorage(SkillStorage):
await _scan_skill_archive_contents_or_raise(skill_dir, skill_name)
with tempfile.TemporaryDirectory(prefix=f".installing-{skill_name}-", dir=custom_dir) as staging_root:
with tempfile.TemporaryDirectory(prefix=f".installing-{skill_name}-", dir=category_dir) as staging_root:
staging_target = Path(staging_root) / skill_name
shutil.copytree(skill_dir, staging_target)
_move_staged_skill_into_reserved_target(staging_target, target)
@@ -242,6 +245,7 @@ class LocalSkillStorage(SkillStorage):
"skill_name": skill_name,
"message": f"Skill '{skill_name}' installed successfully",
"overwritten": replaced,
"category": target_category.value,
}
def delete_custom_skill(self, name: str, *, history_meta: dict | None = None) -> None:
@@ -132,17 +132,28 @@ class SkillStorage(ABC):
"""
@abstractmethod
async def ainstall_skill_from_archive(self, archive_path: str | Path) -> dict:
async def ainstall_skill_from_archive(
self,
archive_path: str | Path,
*,
target_category: SkillCategory = SkillCategory.CUSTOM,
) -> dict:
"""Async install of a skill from a ``.skill`` ZIP archive.
``target_category`` controls the destination directory (public or custom).
Origin: ``deerflow.skills.installer.ainstall_skill_from_archive``.
"""
def install_skill_from_archive(self, archive_path: str | Path) -> dict:
def install_skill_from_archive(
self,
archive_path: str | Path,
*,
target_category: SkillCategory = SkillCategory.CUSTOM,
) -> dict:
"""Sync wrapper — delegates to :meth:`ainstall_skill_from_archive`."""
from deerflow.skills.installer import _run_async_install
return _run_async_install(self.ainstall_skill_from_archive(archive_path))
return _run_async_install(self.ainstall_skill_from_archive(archive_path, target_category=target_category))
@abstractmethod
def delete_custom_skill(self, name: str, *, history_meta: dict | None = None) -> None:
@@ -0,0 +1,159 @@
"""Skill usage sidecar — per-skill telemetry stored in .usage.json."""
from __future__ import annotations
import json
import logging
import os
import tempfile
from datetime import UTC, datetime
from pathlib import Path
logger = logging.getLogger(__name__)
_USAGE_FILENAME = ".usage.json"
STATE_ACTIVE = "active"
STATE_STALE = "stale"
STATE_ARCHIVED = "archived"
SOURCE_AGENT = "agent"
SOURCE_UPLOAD = "upload"
_DEFAULT_ENTRY: dict = {
"use_count": 0,
"view_count": 0,
"patch_count": 0,
"last_used_at": None,
"last_viewed_at": None,
"last_patched_at": None,
"created_at": None,
"state": STATE_ACTIVE,
"pinned": False,
"archived_at": None,
"source": SOURCE_UPLOAD,
}
def _usage_path() -> Path:
from deerflow.skills.storage import get_or_new_skill_storage
return Path(get_or_new_skill_storage().get_skills_root_path()) / _USAGE_FILENAME
def _read_all() -> dict:
path = _usage_path()
if not path.exists():
return {}
try:
return json.loads(path.read_text(encoding="utf-8"))
except Exception:
logger.debug("Failed to read %s", path, exc_info=True)
return {}
def _write_all(data: dict) -> None:
path = _usage_path()
try:
path.parent.mkdir(parents=True, exist_ok=True)
tmp_fd, tmp_path = tempfile.mkstemp(dir=path.parent, suffix=".tmp")
try:
with os.fdopen(tmp_fd, "w", encoding="utf-8") as f:
json.dump(data, f, indent=2, default=str)
os.replace(tmp_path, path)
except Exception:
try:
os.unlink(tmp_path)
except Exception:
pass
raise
except Exception:
logger.debug("Failed to write %s", path, exc_info=True)
def _now_iso() -> str:
return datetime.now(UTC).isoformat()
def _update_entry(name: str, updates: dict) -> None:
try:
data = _read_all()
entry = {**_DEFAULT_ENTRY, **data.get(name, {})}
entry.update(updates)
data[name] = entry
_write_all(data)
except Exception:
logger.debug("Failed to update usage entry for %r", name, exc_info=True)
def get_entry(name: str) -> dict:
"""Return the usage entry for *name*, filling missing fields with defaults."""
data = _read_all()
return {**_DEFAULT_ENTRY, **data.get(name, {})}
def all_entries() -> dict:
"""Return all usage entries keyed by skill name."""
return _read_all()
def agent_created_entries() -> dict:
"""Return only entries where source == 'agent' (eligible for curation)."""
return {k: v for k, v in _read_all().items() if v.get("source") == SOURCE_AGENT}
def bump_view(name: str) -> None:
entry = get_entry(name)
_update_entry(name, {"view_count": entry.get("view_count", 0) + 1, "last_viewed_at": _now_iso()})
def bump_use(name: str) -> None:
entry = get_entry(name)
_update_entry(name, {"use_count": entry.get("use_count", 0) + 1, "last_used_at": _now_iso()})
def bump_patch(name: str) -> None:
entry = get_entry(name)
_update_entry(name, {"patch_count": entry.get("patch_count", 0) + 1, "last_patched_at": _now_iso()})
def set_state(name: str, state: str) -> None:
updates: dict = {"state": state}
if state == STATE_ARCHIVED:
updates["archived_at"] = _now_iso()
elif state == STATE_ACTIVE:
updates["archived_at"] = None
_update_entry(name, updates)
def set_pinned(name: str, pinned: bool) -> None:
_update_entry(name, {"pinned": pinned})
def forget(name: str) -> None:
try:
data = _read_all()
data.pop(name, None)
_write_all(data)
except Exception:
logger.debug("Failed to forget usage entry for %r", name, exc_info=True)
def mark_agent_created(name: str) -> None:
"""Record a skill as agent-created; sets source='agent' and initialises created_at."""
_update_entry(name, {"source": SOURCE_AGENT, "created_at": _now_iso(), "state": STATE_ACTIVE})
def mark_uploaded(name: str) -> None:
"""Record a skill as user-uploaded; sets source='upload'."""
_update_entry(name, {"source": SOURCE_UPLOAD, "created_at": _now_iso()})
def latest_activity_at(name: str) -> str | None:
"""Return the most recent activity timestamp, falling back to created_at."""
entry = get_entry(name)
candidates = [entry.get("last_used_at"), entry.get("last_viewed_at"), entry.get("last_patched_at")]
valid = [ts for ts in candidates if ts]
if valid:
return max(valid)
return entry.get("created_at")
@@ -0,0 +1,147 @@
"""skill_view and skill_list — read-only tools for agent skill introspection."""
from __future__ import annotations
import json
import logging
from langchain.tools import ToolRuntime, tool
logger = logging.getLogger(__name__)
def _resolve_active_agent_id(runtime: ToolRuntime | None) -> str | None:
"""Extract the agent_id under which the current tool call is running.
Custom agents pass their id via either the ``context`` dict or the
``configurable`` section of the LangGraph runtime config. The default
lead agent runs without an explicit agent_id — callers should treat
a None return as "no per-agent skills filter applies".
"""
if runtime is None:
return None
try:
if getattr(runtime, "context", None):
ctx_id = runtime.context.get("agent_id")
if ctx_id:
return str(ctx_id)
cfg = getattr(runtime, "config", None)
if isinstance(cfg, dict):
cid = cfg.get("configurable", {}).get("agent_id")
if cid:
return str(cid)
except Exception:
return None
return None
def _load_agent_skill_whitelist(agent_id: str | None) -> set[str] | None:
"""Return the explicit skill name set declared in an agent's config.yaml.
Semantics:
- ``None`` returned → no agent_id, or config missing — caller should
fall back to "all skills visible".
- ``set()`` (empty) → agent has ``skills: []`` declared — visibility
is intentionally empty.
- Non-empty set → only these names are visible to this agent.
"""
if not agent_id:
return None
try:
from deerflow.config.agents_config import load_agent_config
cfg = load_agent_config(agent_id)
if cfg is None or cfg.skills is None:
return None
return set(cfg.skills)
except Exception:
logger.debug("Failed to load agent config for skill_list filter", exc_info=True)
return None
@tool("skill_view")
def skill_view_tool(name: str) -> str:
"""Read the full SKILL.md content of a skill by name.
Args:
name: The skill name (hyphen-case, e.g. 'my-skill').
"""
from deerflow.skills.storage import get_or_new_skill_storage
from deerflow.skills.usage import bump_view
storage = get_or_new_skill_storage()
try:
if storage.custom_skill_exists(name):
content = storage.read_custom_skill(name)
elif storage.public_skill_exists(name):
skill_file = storage.get_skills_root_path()
from pathlib import Path
from deerflow.skills.types import SKILL_MD_FILE
path = Path(skill_file) / "public" / name / SKILL_MD_FILE
if not path.exists():
return f"Skill '{name}' not found."
content = path.read_text(encoding="utf-8")
else:
return f"Skill '{name}' not found."
except Exception as e:
logger.debug("skill_view failed for %r: %s", name, e)
return f"Error reading skill '{name}': {e}"
try:
bump_view(name)
except Exception:
pass
return content
@tool("skill_list")
def skill_list_tool(runtime: ToolRuntime) -> str:
"""List skills available to this agent, with metadata and usage stats.
Does NOT read SKILL.md content.
For a **custom agent**, the result is filtered to skills explicitly
declared in the agent's ``.deer-flow/agents/{agent_id}/config.yaml``
under the ``skills:`` field — the agent only sees what it was wired to
use.
For the **default lead agent** (no agent_id in runtime), there is no
per-agent whitelist and every visible skill is returned.
Returns a JSON array with fields: name, description, category, enabled,
state, use_count, view_count, patch_count, last_activity_at, pinned,
source.
"""
from deerflow.skills.storage import get_or_new_skill_storage
from deerflow.skills.usage import all_entries, latest_activity_at
agent_id = _resolve_active_agent_id(runtime)
allowed = _load_agent_skill_whitelist(agent_id)
try:
storage = get_or_new_skill_storage()
skills = storage.load_skills(enabled_only=False)
usage = all_entries()
result = []
for skill in skills:
if allowed is not None and skill.name not in allowed:
continue
entry = usage.get(skill.name, {})
result.append({
"name": skill.name,
"description": skill.description,
"category": skill.category.value if hasattr(skill.category, "value") else str(skill.category),
"enabled": skill.enabled,
"state": entry.get("state", "active"),
"use_count": entry.get("use_count", 0),
"view_count": entry.get("view_count", 0),
"patch_count": entry.get("patch_count", 0),
"last_activity_at": latest_activity_at(skill.name),
"pinned": entry.get("pinned", False),
"source": entry.get("source", "upload"),
})
return json.dumps(result, ensure_ascii=False, indent=2)
except Exception as e:
logger.debug("skill_list failed: %s", e)
return f"Error listing skills: {e}"
@@ -13,6 +13,8 @@ from langgraph.typing import ContextT
from deerflow.agents.lead_agent.prompt import refresh_skills_system_prompt_cache_async
from deerflow.agents.thread_state import ThreadState
from deerflow.mcp.tools import _make_sync_tool_wrapper
from deerflow.persistence.skills import SkillStore
from deerflow.runtime.user_context import get_effective_user_id
from deerflow.skills.security_scanner import scan_skill_content
from deerflow.skills.storage import get_or_new_skill_storage
from deerflow.skills.storage.skill_storage import SkillStorage
@@ -64,6 +66,85 @@ async def _to_thread(func, /, *args, **kwargs):
return await asyncio.to_thread(func, *args, **kwargs)
async def _ensure_skill_db_registered(store: SkillStore, name: str, user_id: str) -> None:
"""Idempotently record an agent-created skill's ownership in the DB.
Mirrors the ``/api/skills/install-upload`` path so agent-created skills
show up in the Web UI listing (which filters custom skills by DB
ownership). ``ensure_legacy`` is idempotent — it never overwrites an
existing owner — so re-running the create flow on an already-owned skill
is a no-op. Failures are swallowed so DB outages can never break the
agent's create flow itself.
"""
try:
await store.ensure_legacy(
{"name": name, "owner_user_id": user_id, "published": False}
)
except Exception:
logger.debug("Failed to register skill ownership for %r", name, exc_info=True)
def _default_skill_store() -> SkillStore | None:
"""Build a ``SkillStore`` backed by the global session factory.
Returns ``None`` if no DB session factory is configured (e.g. ephemeral
test environments), letting the caller skip the registration step
silently.
"""
try:
from deerflow.persistence.engine import get_session_factory
from deerflow.persistence.skills import make_skill_store
factory = get_session_factory()
if factory is None:
return None
return make_skill_store(factory)
except Exception:
logger.debug("Failed to build default skill store", exc_info=True)
return None
def _default_agent_store() -> Any | None:
"""Build an ``AgentStore`` backed by the global session factory."""
try:
from deerflow.persistence.agents import make_agent_store
from deerflow.persistence.engine import get_session_factory
factory = get_session_factory()
if factory is None:
return None
return make_agent_store(factory)
except Exception:
logger.debug("Failed to build default agent store", exc_info=True)
return None
async def _assert_agent_may_delete(store: SkillStore, name: str, user_id: str) -> dict[str, Any]:
"""Pre-flight ownership check before agent-delete touches the on-disk file.
LocalSkillStorage doesn't know about users — it deletes anything that
matches by name. Without this guard, a malicious or hallucinating agent
could destroy another user's skill just by knowing the name. We refuse
unless the DB confirms ``user_id`` owns the skill outright.
Returns the DB record for use in the subsequent cascade step. Raises
``ValueError`` (mapped to a tool error visible to the LLM) on refusal.
"""
record = await store.get_any(name)
if record is None:
# No DB row — could be a legacy/orphan skill. Refuse rather than
# delete blindly; admin can clean up via the Web UI.
raise ValueError(
f"Refused: skill '{name}' has no ownership record. Ask a human to delete it via the Web UI."
)
owner = record.get("owner_user_id")
if owner is None:
raise ValueError(f"Refused: skill '{name}' is built-in and cannot be deleted by an agent.")
if owner != user_id:
raise ValueError(f"Refused: skill '{name}' is owned by another user.")
return record
async def _skill_manage_impl(
runtime: ToolRuntime[ContextT, ThreadState],
action: str,
@@ -78,6 +159,10 @@ async def _skill_manage_impl(
Args:
action: One of create, patch, edit, delete, write_file, remove_file.
For listing or viewing skills, prefer the dedicated
``skill_list`` / ``skill_view`` tools — but ``action='list'``
and ``action='view'`` are accepted here as convenience
aliases that delegate to those tools.
name: Skill name in hyphen-case.
content: New file content for create, edit, or write_file.
path: Supporting file path for write_file or remove_file.
@@ -85,6 +170,17 @@ async def _skill_manage_impl(
replace: Replacement text for patch.
expected_count: Optional expected number of replacements for patch.
"""
# Read-only aliases — delegate to the dedicated tools so a model that
# over-uses skill_manage doesn't trip on Unsupported action errors.
if action == "list":
from deerflow.tools.builtins.skill_tools import skill_list_tool
return await _to_thread(skill_list_tool.invoke, {})
if action == "view":
from deerflow.tools.builtins.skill_tools import skill_view_tool
return await _to_thread(skill_view_tool.invoke, {"name": name})
name = SkillStorage.validate_skill_name(name)
lock = _get_lock(name)
thread_id = _get_thread_id(runtime)
@@ -104,6 +200,18 @@ async def _skill_manage_impl(
name,
_history_record(action="create", file_path=SKILL_MD_FILE, prev_content=None, new_content=content, thread_id=thread_id, scanner=scan),
)
try:
from deerflow.skills.usage import mark_agent_created
await _to_thread(mark_agent_created, name)
except Exception:
pass
# Register the new skill in the DB so the web UI listing actually
# surfaces it (custom skills without a DB row are filtered out by
# ``GET /api/skills``). Resolve the user_id *here* on the main
# async stack — contextvars don't propagate into ``to_thread``.
store = _default_skill_store()
if store is not None:
await _ensure_skill_db_registered(store, name, get_effective_user_id())
await refresh_skills_system_prompt_cache_async()
return f"Created custom skill '{name}'."
@@ -149,6 +257,15 @@ async def _skill_manage_impl(
return f"Patched custom skill '{name}' ({replacement_count} replacement(s) applied, {occurrences} match(es) found)."
if action == "delete":
# Ownership pre-check: LocalSkillStorage deletes purely by name,
# so without this an agent could blow away another user's skill
# just by knowing the name. Raise BEFORE touching the filesystem.
user_id = get_effective_user_id()
store = _default_skill_store()
record: dict[str, Any] | None = None
if store is not None:
record = await _assert_agent_may_delete(store, name, user_id)
await _to_thread(
skill_storage.delete_custom_skill,
name,
@@ -161,6 +278,36 @@ async def _skill_manage_impl(
scanner={"decision": "allow", "reason": "Deletion requested."},
),
)
# Mirror what ``DELETE /api/skills/custom/{name}`` does: drop the
# ``.usage.json`` entry and the DB ownership row, then cascade
# the removal to every agent that referenced this skill so
# ``config.yaml`` doesn't end up with a dangling reference.
# Failures are best-effort — the on-disk delete already happened
# and is authoritative.
try:
from deerflow.skills.usage import forget
await _to_thread(forget, name)
except Exception:
logger.debug("Failed to forget usage entry for %r", name, exc_info=True)
if store is not None:
try:
await store.delete(name, user_id)
except Exception:
logger.debug("Failed to delete DB row for %r", name, exc_info=True)
agent_store = _default_agent_store()
if agent_store is not None:
try:
from deerflow.skills.cascade import cascade_skill_unpublished_or_deleted
await cascade_skill_unpublished_or_deleted(
name,
actor_user_id=user_id,
operation="delete",
skill_owner_user_id=(record.get("owner_user_id") if record else user_id),
agent_store=agent_store,
)
except Exception:
logger.debug("Cascade failed for deleted skill %r", name, exc_info=True)
await refresh_skills_system_prompt_cache_async()
return f"Deleted custom skill '{name}'."
@@ -86,9 +86,10 @@ def get_available_tools(
builtin_tools.append(scheduled_task_tool)
skill_evolution_config = getattr(config, "skill_evolution", None)
if getattr(skill_evolution_config, "enabled", False):
from deerflow.tools.builtins.skill_tools import skill_list_tool, skill_view_tool
from deerflow.tools.skill_manage_tool import skill_manage_tool
builtin_tools.append(skill_manage_tool)
builtin_tools.extend([skill_list_tool, skill_view_tool, skill_manage_tool])
# Add subagent tools only if enabled via runtime parameter
if subagent_enabled: