# Conflicts:
#	frontend-web/src/pages/WorkspaceRoutes.tsx
#	offline-backend-20260512/backend/packages/harness/deerflow/persistence/models/__init__.py
This commit is contained in:
吉鑫升 committed 2026-05-15 11:21:48 +08:00
commit aeba15eb16
88 files changed
+13824 -3572

No files matched your search

@@ -20,6 +20,8 @@ from deerflow.config.run_events_config import RunEventsConfig
from deerflow.config.runtime_paths import existing_project_file
from deerflow.config.sandbox_config import SandboxConfig
from deerflow.config.skill_evolution_config import SkillEvolutionConfig
from deerflow.config.skill_evolution_runtime import get_overrides_mtime as _get_skill_evolution_overrides_mtime
from deerflow.config.skill_evolution_runtime import load_overrides as _load_skill_evolution_overrides
from deerflow.config.skills_config import SkillsConfig
from deerflow.config.stream_bridge_config import StreamBridgeConfig, load_stream_bridge_config_from_dict
from deerflow.config.subagents_config import SubagentsAppConfig, load_subagents_config_from_dict
@@ -54,6 +56,22 @@ def _legacy_config_candidates() -> tuple[Path, ...]:
return (backend_dir / "config.yaml", repo_root / "config.yaml")
def _deep_merge_dict(base: dict[str, Any], overrides: dict[str, Any]) -> dict[str, Any]:
"""Deep-merge ``overrides`` onto a copy of ``base`` and return the result.
Nested dicts are merged recursively; any other value type in ``overrides``
replaces the corresponding value in ``base`` outright.
"""
merged: dict[str, Any] = dict(base)
for key, value in overrides.items():
existing = merged.get(key)
if isinstance(existing, dict) and isinstance(value, dict):
merged[key] = _deep_merge_dict(existing, value)
else:
merged[key] = value
return merged
def logging_level_from_config(name: str | None) -> int:
"""Map ``config.yaml`` ``log_level`` string to a :mod:`logging` level constant."""
mapping = logging.getLevelNamesMapping()
@@ -204,6 +222,15 @@ class AppConfig(BaseModel):
extensions_config = ExtensionsConfig.from_file()
config_data["extensions"] = extensions_config.model_dump()
# Apply skill_evolution runtime overrides from skill_evolution.json
# (file is optional; load_overrides returns None if missing/invalid)
overrides = _load_skill_evolution_overrides()
if overrides:
yaml_section = config_data.get("skill_evolution") or {}
if not isinstance(yaml_section, dict):
yaml_section = {}
config_data["skill_evolution"] = _deep_merge_dict(yaml_section, overrides)
result = cls.model_validate(config_data)
return result
@@ -329,6 +356,7 @@ class AppConfig(BaseModel):
_app_config: AppConfig | None = None
_app_config_path: Path | None = None
_app_config_mtime: float | None = None
_app_config_skill_evolution_overrides_mtime: float | None = None
_app_config_is_custom = False
_current_app_config: ContextVar[AppConfig | None] = ContextVar("deerflow_current_app_config", default=None)
_current_app_config_stack: ContextVar[tuple[AppConfig | None, ...]] = ContextVar("deerflow_current_app_config_stack", default=())
@@ -344,12 +372,13 @@ def _get_config_mtime(config_path: Path) -> float | None:
def _load_and_cache_app_config(config_path: str | None = None) -> AppConfig:
"""Load config from disk and refresh cache metadata."""
global _app_config, _app_config_path, _app_config_mtime, _app_config_is_custom
global _app_config, _app_config_path, _app_config_mtime, _app_config_skill_evolution_overrides_mtime, _app_config_is_custom
resolved_path = AppConfig.resolve_config_path(config_path)
_app_config = AppConfig.from_file(str(resolved_path))
_app_config_path = resolved_path
_app_config_mtime = _get_config_mtime(resolved_path)
_app_config_skill_evolution_overrides_mtime = _get_skill_evolution_overrides_mtime()
_app_config_is_custom = False
return _app_config
@@ -373,8 +402,14 @@ def get_app_config() -> AppConfig:
resolved_path = AppConfig.resolve_config_path()
current_mtime = _get_config_mtime(resolved_path)
current_overrides_mtime = _get_skill_evolution_overrides_mtime()
should_reload = _app_config is None or _app_config_path != resolved_path or _app_config_mtime != current_mtime
should_reload = (
_app_config is None
or _app_config_path != resolved_path
or _app_config_mtime != current_mtime
or _app_config_skill_evolution_overrides_mtime != current_overrides_mtime
)
if should_reload:
if _app_config_path == resolved_path and _app_config_mtime is not None and current_mtime is not None and _app_config_mtime != current_mtime:
logger.info(
@@ -382,6 +417,12 @@ def get_app_config() -> AppConfig:
_app_config_mtime,
current_mtime,
)
elif _app_config_skill_evolution_overrides_mtime != current_overrides_mtime:
logger.info(
"skill_evolution.json overrides changed (mtime: %s -> %s), reloading AppConfig",
_app_config_skill_evolution_overrides_mtime,
current_overrides_mtime,
)
_load_and_cache_app_config(str(resolved_path))
return _app_config
@@ -409,10 +450,11 @@ def reset_app_config() -> None:
`get_app_config()` to reload from file. Useful for testing
or when switching between different configurations.
"""
global _app_config, _app_config_path, _app_config_mtime, _app_config_is_custom
global _app_config, _app_config_path, _app_config_mtime, _app_config_skill_evolution_overrides_mtime, _app_config_is_custom
_app_config = None
_app_config_path = None
_app_config_mtime = None
_app_config_skill_evolution_overrides_mtime = None
_app_config_is_custom = False
@@ -424,10 +466,11 @@ def set_app_config(config: AppConfig) -> None:
Args:
config: The AppConfig instance to use.
"""
global _app_config, _app_config_path, _app_config_mtime, _app_config_is_custom
global _app_config, _app_config_path, _app_config_mtime, _app_config_skill_evolution_overrides_mtime, _app_config_is_custom
_app_config = config
_app_config_path = None
_app_config_mtime = None
_app_config_skill_evolution_overrides_mtime = None
_app_config_is_custom = True
@@ -1,6 +1,15 @@
from pydantic import BaseModel, Field
class CuratorConfig(BaseModel):
"""Configuration for the background skill curator."""
enabled: bool = Field(default=True, description="Whether the curator runs automatically.")
interval_hours: int = Field(default=168, description="Hours between curator runs (default: 7 days).")
stale_after_days: int = Field(default=30, description="Days of inactivity before a skill is marked stale.")
archive_after_days: int = Field(default=90, description="Days of inactivity before a stale skill is archived.")
class SkillEvolutionConfig(BaseModel):
"""Configuration for agent-managed skill evolution."""
@@ -12,3 +21,7 @@ class SkillEvolutionConfig(BaseModel):
default=None,
description="Optional model name for skill security moderation. Defaults to the primary chat model.",
)
curator: CuratorConfig = Field(
default_factory=CuratorConfig,
description="Configuration for the background skill curator.",
)
@@ -0,0 +1,216 @@
"""Runtime overrides for `skill_evolution` config, stored in skill_evolution.json.
Two scopes exist:
* **Global** (back-compat): ``backend/skill_evolution.json`` deep-merged onto
``config.yaml.skill_evolution`` at AppConfig load time. Used as a fallback
for users who never set a personal override.
* **Per-user**: ``.deer-flow/users/{user_id}/skill_evolution.json``. Read by
the curator on each run and by the ``GET/PUT /api/curator/config`` route
with the active user id. Curator merges this on top of the global config
via :func:`load_curator_config_for_user`.
The file is optional in both scopes; absence (or a malformed payload) is
treated as "no overrides" so the system keeps the YAML-only behaviour.
"""
from __future__ import annotations
import json
import logging
import os
import tempfile
from pathlib import Path
from typing import Any
from deerflow.config.runtime_paths import existing_project_file
logger = logging.getLogger(__name__)
_FILENAME = "skill_evolution.json"
def resolve_path() -> Path:
"""Resolve the on-disk location of ``skill_evolution.json``.
Priority:
1. ``DEER_FLOW_SKILL_EVOLUTION_CONFIG_PATH`` environment variable
2. ``skill_evolution.json`` in the project root (current and parents)
3. Legacy ``backend/`` or repo-root sibling locations
4. Falls back to ``backend/skill_evolution.json`` (does not need to exist)
"""
override = os.getenv("DEER_FLOW_SKILL_EVOLUTION_CONFIG_PATH")
if override:
return Path(override)
project_file = existing_project_file((_FILENAME,))
if project_file is not None:
return project_file
backend_dir = Path(__file__).resolve().parents[4]
repo_root = backend_dir.parent
for candidate in (backend_dir / _FILENAME, repo_root / _FILENAME):
if candidate.exists():
return candidate
return backend_dir / _FILENAME
def load_overrides() -> dict[str, Any] | None:
"""Read overrides from disk.
Returns ``None`` when the file does not exist, is empty, contains
non-dict JSON, or fails to parse. Errors are logged at WARNING level
so production stays silent in the common "no overrides" case.
"""
path = resolve_path()
if not path.exists():
return None
try:
raw = path.read_text(encoding="utf-8").strip()
if not raw:
return None
data = json.loads(raw)
except OSError as exc:
logger.warning("Failed to read %s: %s", path, exc)
return None
except json.JSONDecodeError as exc:
logger.warning("Skill evolution overrides at %s are not valid JSON: %s", path, exc)
return None
if not isinstance(data, dict):
logger.warning("Skill evolution overrides at %s must be a JSON object, got %s", path, type(data).__name__)
return None
return data
def save_overrides(payload: dict[str, Any]) -> Path:
"""Atomically persist ``payload`` to ``skill_evolution.json``.
The caller is responsible for shape validation (e.g. via the
``SkillEvolutionConfig`` Pydantic model). Returns the path that was
written so callers can log it.
"""
if not isinstance(payload, dict):
raise TypeError("skill_evolution overrides payload must be a dict")
path = resolve_path()
path.parent.mkdir(parents=True, exist_ok=True)
fd, tmp_path_str = tempfile.mkstemp(prefix=".skill-evolution-", suffix=".tmp", dir=str(path.parent))
tmp_path = Path(tmp_path_str)
try:
with os.fdopen(fd, "w", encoding="utf-8") as f:
json.dump(payload, f, indent=2, ensure_ascii=False)
os.replace(tmp_path, path)
except Exception:
tmp_path.unlink(missing_ok=True)
raise
logger.info("Wrote skill_evolution overrides to %s", path)
return path
def get_overrides_mtime() -> float | None:
"""Return the mtime of ``skill_evolution.json`` if it exists, else None.
Used by the AppConfig cache to decide whether to reload — when the
file appears, disappears, or is modified, the mtime tuple changes
and ``get_app_config()`` rebuilds.
"""
path = resolve_path()
try:
return path.stat().st_mtime
except OSError:
return None
# ---------------------------------------------------------------------------
# Per-user overrides
# ---------------------------------------------------------------------------
def resolve_user_path(user_id: str) -> Path:
"""Return ``{base_dir}/users/{user_id}/skill_evolution.json``."""
from deerflow.config.paths import get_paths
return get_paths().user_dir(user_id) / _FILENAME
def load_user_overrides(user_id: str) -> dict[str, Any] | None:
"""Read the per-user overrides file for ``user_id`` (None if missing/invalid)."""
path = resolve_user_path(user_id)
if not path.exists():
return None
try:
raw = path.read_text(encoding="utf-8").strip()
if not raw:
return None
data = json.loads(raw)
except OSError as exc:
logger.warning("Failed to read %s: %s", path, exc)
return None
except json.JSONDecodeError as exc:
logger.warning("User skill_evolution overrides at %s are not valid JSON: %s", path, exc)
return None
if not isinstance(data, dict):
logger.warning("User skill_evolution overrides at %s must be a JSON object", path)
return None
return data
def save_user_overrides(user_id: str, payload: dict[str, Any]) -> Path:
"""Atomically persist ``payload`` to the per-user overrides file."""
if not isinstance(payload, dict):
raise TypeError("skill_evolution overrides payload must be a dict")
path = resolve_user_path(user_id)
path.parent.mkdir(parents=True, exist_ok=True)
fd, tmp_path_str = tempfile.mkstemp(prefix=".skill-evolution-", suffix=".tmp", dir=str(path.parent))
tmp_path = Path(tmp_path_str)
try:
with os.fdopen(fd, "w", encoding="utf-8") as f:
json.dump(payload, f, indent=2, ensure_ascii=False)
os.replace(tmp_path, path)
except Exception:
tmp_path.unlink(missing_ok=True)
raise
logger.info("Wrote per-user skill_evolution overrides to %s", path)
return path
def get_user_overrides_mtime(user_id: str) -> float | None:
path = resolve_user_path(user_id)
try:
return path.stat().st_mtime
except OSError:
return None
def load_skill_evolution_config_for_user(user_id: str | None):
"""Return a ``SkillEvolutionConfig`` instance with per-user overrides merged
on top of the global AppConfig defaults.
Used by the curator + the per-user ``GET/PUT /api/curator/config`` route.
When ``user_id`` is None or the per-user file is absent, returns the global
config unchanged.
"""
from deerflow.config import get_app_config
from deerflow.config.skill_evolution_config import SkillEvolutionConfig
base = get_app_config().skill_evolution
if not user_id:
return base
overrides = load_user_overrides(user_id)
if not overrides:
return base
merged = base.model_dump()
for key, value in overrides.items():
existing = merged.get(key)
if isinstance(existing, dict) and isinstance(value, dict):
merged[key] = {**existing, **value}
else:
merged[key] = value
try:
return SkillEvolutionConfig.model_validate(merged)
except Exception:
logger.warning("Per-user skill_evolution overrides for %s failed validation, ignoring", user_id, exc_info=True)
return base