Files
deerflow-code/offline-backend-20260512/backend/packages/harness/deerflow/skills/usage.py
T

326 lines
12 KiB
Python

"""Skill usage sidecar — per-skill telemetry stored in .usage.json."""
from __future__ import annotations
import json
import logging
import os
import tempfile
import threading
from collections import OrderedDict
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
logger = logging.getLogger(__name__)
_USAGE_FILENAME = ".usage.json"
# 串行化对 .usage.json 的"读-改-写",避免并发(尤其 curator 多线程)丢更新。
# .usage.json 是全局单文件;此锁覆盖进程内并发。leader 选举已使 curator 只在
# 单进程跑,故进程内锁即可保证状态流转不丢。
_usage_lock = threading.Lock()
# One user run can first load SKILL.md and then execute several helper scripts.
# Those are implementation details of one logical skill invocation, so retain a
# small process-local LRU keyed by (run, skill) and increment the public counter
# only once. A run stays on one worker; persistence across restarts is neither
# needed nor desirable here.
_MAX_RECENT_USE_EVENTS = 8192
_recent_use_events: OrderedDict[tuple[str, str], None] = OrderedDict()
STATE_ACTIVE = "active"
STATE_STALE = "stale"
STATE_ARCHIVED = "archived"
SOURCE_AGENT = "agent"
SOURCE_UPLOAD = "upload"
_DEFAULT_ENTRY: dict = {
"use_count": 0,
"view_count": 0,
"patch_count": 0,
"last_used_at": None,
"last_viewed_at": None,
"last_patched_at": None,
"created_at": None,
"state": STATE_ACTIVE,
"pinned": False,
"featured": False,
"archived_at": None,
"source": SOURCE_UPLOAD,
}
def _usage_path() -> Path:
from deerflow.skills.storage import get_or_new_skill_storage
return Path(get_or_new_skill_storage().get_skills_root_path()) / _USAGE_FILENAME
def _read_all() -> dict:
path = _usage_path()
if not path.exists():
return {}
try:
return json.loads(path.read_text(encoding="utf-8"))
except Exception:
logger.debug("Failed to read %s", path, exc_info=True)
return {}
def _write_all(data: dict) -> None:
path = _usage_path()
try:
path.parent.mkdir(parents=True, exist_ok=True)
tmp_fd, tmp_path = tempfile.mkstemp(dir=path.parent, suffix=".tmp")
try:
with os.fdopen(tmp_fd, "w", encoding="utf-8") as f:
json.dump(data, f, indent=2, default=str)
os.replace(tmp_path, path)
except Exception:
try:
os.unlink(tmp_path)
except Exception:
pass
raise
except Exception:
logger.debug("Failed to write %s", path, exc_info=True)
def _now_iso() -> str:
return datetime.now(UTC).isoformat()
def _update_entry(name: str, updates) -> None:
"""更新一个 usage 条目。``updates`` 可以是 dict,或 ``(entry) -> dict`` 回调。
用回调形式可让"读当前值 -> 算增量"也落在锁内(例如计数自增),
避免并发下的 lost update。
"""
try:
with _usage_lock:
data = _read_all()
entry = {**_DEFAULT_ENTRY, **data.get(name, {})}
resolved = updates(entry) if callable(updates) else updates
entry.update(resolved)
data[name] = entry
_write_all(data)
except Exception:
logger.debug("Failed to update usage entry for %r", name, exc_info=True)
def get_entry(name: str) -> dict:
"""Return the usage entry for *name*, filling missing fields with defaults."""
data = _read_all()
return {**_DEFAULT_ENTRY, **data.get(name, {})}
def all_entries() -> dict:
"""Return all usage entries keyed by skill name."""
return _read_all()
def agent_created_entries() -> dict:
"""Return only entries where source == 'agent' (eligible for curation)."""
return {k: v for k, v in _read_all().items() if v.get("source") == SOURCE_AGENT}
def bump_view(name: str) -> None:
# 用回调:读当前计数 + 自增在同一把锁内完成,避免并发丢计数。
_update_entry(name, lambda e: {"view_count": e.get("view_count", 0) + 1, "last_viewed_at": _now_iso()})
async def _increment_use_projection(name: str, use_count: int, used_at: datetime) -> None:
"""Keep the MySQL skill-list projection current without importing ``app``."""
try:
from sqlalchemy import case, update
from deerflow.persistence.engine import get_session_factory
from deerflow.persistence.skills.model import SkillRow
session_factory = get_session_factory()
if session_factory is None:
return
async with session_factory() as session:
await session.execute(
update(SkillRow)
.where(SkillRow.name == name)
.values(
# Fire-and-forget writes may finish out of order. Never let
# an older absolute sidecar count overwrite a newer value.
use_count=case((SkillRow.use_count < use_count, use_count), else_=SkillRow.use_count),
last_activity_at=case(
(SkillRow.last_activity_at.is_(None), used_at),
(SkillRow.last_activity_at < used_at, used_at),
else_=SkillRow.last_activity_at,
),
updated_at=case(
(SkillRow.updated_at < used_at, used_at),
else_=SkillRow.updated_at,
),
)
)
await session.commit()
except Exception:
logger.debug("Failed to update skill usage projection for %r", name, exc_info=True)
def _schedule_use_projection_increment(name: str, use_count: int, used_at: datetime) -> None:
"""Best-effort bridge from synchronous sandbox tools to the DB home loop."""
try:
from deerflow.persistence.engine import fire_and_forget_db_write
fire_and_forget_db_write(_increment_use_projection(name, use_count, used_at))
except Exception:
logger.debug("Failed to schedule skill usage projection for %r", name, exc_info=True)
def runtime_use_event_key(runtime: Any | None) -> str | None:
"""Return the stable user-run identifier carried by ``ToolRuntime``."""
if runtime is None:
return None
context = getattr(runtime, "context", None)
if isinstance(context, dict):
for key in ("run_id", "langgraph_run_id"):
value = context.get(key)
if value is not None and str(value).strip():
return str(value).strip()
config = getattr(runtime, "config", None)
if isinstance(config, dict):
for bucket_name in ("configurable", "metadata"):
bucket = config.get(bucket_name)
if not isinstance(bucket, dict):
continue
for key in ("run_id", "langgraph_run_id"):
value = bucket.get(key)
if value is not None and str(value).strip():
return str(value).strip()
return None
def bump_use(name: str, *, event_key: str | None = None) -> bool:
"""Count one logical use, optionally de-duplicated by user run.
Returns ``True`` only when the counter changed. This makes callers and tests
able to distinguish a new invocation from a second file/script access in
the same run.
"""
normalized_name = str(name or "").strip()
if not normalized_name:
return False
normalized_event = str(event_key or "").strip() or None
dedupe_key = (normalized_event, normalized_name) if normalized_event else None
used_at = datetime.now(UTC)
new_count = 0
try:
with _usage_lock:
if dedupe_key is not None and dedupe_key in _recent_use_events:
_recent_use_events.move_to_end(dedupe_key)
return False
data = _read_all()
entry = {**_DEFAULT_ENTRY, **data.get(normalized_name, {})}
new_count = max(0, int(entry.get("use_count") or 0)) + 1
entry.update({"use_count": new_count, "last_used_at": used_at.isoformat()})
data[normalized_name] = entry
_write_all(data)
if dedupe_key is not None:
_recent_use_events[dedupe_key] = None
_recent_use_events.move_to_end(dedupe_key)
while len(_recent_use_events) > _MAX_RECENT_USE_EVENTS:
_recent_use_events.popitem(last=False)
except Exception:
logger.debug("Failed to bump usage entry for %r", normalized_name, exc_info=True)
return False
_schedule_use_projection_increment(normalized_name, new_count, used_at)
return True
def bump_patch(name: str) -> None:
_update_entry(name, lambda e: {"patch_count": e.get("patch_count", 0) + 1, "last_patched_at": _now_iso()})
def set_state(name: str, state: str) -> None:
updates: dict = {"state": state}
if state == STATE_ARCHIVED:
updates["archived_at"] = _now_iso()
elif state == STATE_ACTIVE:
updates["archived_at"] = None
_update_entry(name, updates)
def set_pinned(name: str, pinned: bool) -> None:
_update_entry(name, {"pinned": pinned})
def set_featured(name: str, featured: bool) -> None:
"""置顶/取消置顶。与 pinned(防归档)正交,仅影响列表排序展示。"""
_update_entry(name, {"featured": featured})
def forget(name: str) -> None:
try:
with _usage_lock:
data = _read_all()
data.pop(name, None)
_write_all(data)
except Exception:
logger.debug("Failed to forget usage entry for %r", name, exc_info=True)
def mark_agent_created(name: str) -> None:
"""Record a skill as agent-created; sets source='agent' and initialises created_at."""
_update_entry(name, {"source": SOURCE_AGENT, "created_at": _now_iso(), "state": STATE_ACTIVE})
def mark_uploaded(name: str) -> None:
"""Record a skill as user-uploaded; sets source='upload'."""
_update_entry(name, {"source": SOURCE_UPLOAD, "created_at": _now_iso()})
def set_migration_metadata_batch(entries: dict[str, dict[str, Any]]) -> None:
"""Write trusted migration metadata for many skills in a single sidecar write."""
if not entries:
return
try:
with _usage_lock:
data = _read_all()
now = _now_iso()
for name, metadata in entries.items():
entry = {**_DEFAULT_ENTRY, **data.get(name, {})}
updates: dict[str, Any] = {}
if "state" in metadata:
state = metadata["state"]
updates["state"] = state
if state == STATE_ARCHIVED:
updates["archived_at"] = entry.get("archived_at") or now
elif state == STATE_ACTIVE:
updates["archived_at"] = None
if "pinned" in metadata:
updates["pinned"] = bool(metadata["pinned"])
if "featured" in metadata:
updates["featured"] = bool(metadata["featured"])
if "source" in metadata and metadata["source"]:
updates["source"] = metadata["source"]
created_at = metadata.get("created_at")
if created_at:
updates["created_at"] = created_at
elif not entry.get("created_at"):
updates["created_at"] = now
entry.update(updates)
data[name] = entry
_write_all(data)
except Exception:
logger.debug("Failed to write migration usage metadata batch", exc_info=True)
def latest_activity_at(name: str) -> str | None:
"""Return the most recent activity timestamp, falling back to created_at."""
entry = get_entry(name)
candidates = [entry.get("last_used_at"), entry.get("last_viewed_at"), entry.get("last_patched_at")]
valid = [ts for ts in candidates if ts]
if valid:
return max(valid)
return entry.get("created_at")