Merge remote-tracking branch 'origin/master' into ji

This commit is contained in:
吉鑫升 committed 2026-05-18 10:03:49 +08:00
commit 4761677676
74 files changed
+9770 -2887

No files matched your search

@@ -1,6 +1,7 @@
import asyncio
import logging
import os
import sys
from collections.abc import AsyncGenerator
from contextlib import asynccontextmanager
@@ -268,6 +269,93 @@ async def _run_db_migrations() -> None:
logger.exception("Database migration failed — schema may be out of date")
# Skill curator 调度参数。
# - 检查间隔:每隔这么久检查一次"需不需要进化";真正是否执行由 should_run_now() 判定。
# - 启动延迟:启动后先等这么久再首次检查,避开启动高峰。
_CURATOR_CHECK_INTERVAL_SECONDS = 3600 # 1 小时
_CURATOR_STARTUP_DELAY_SECONDS = 300 # 5 分钟
def _acquire_singleton_lock(lock_path):
"""跨平台非阻塞独占文件锁,用于多 worker 下的 leader 选举。
成功 -> 返回持有锁的文件句柄(调用方须保持其存活,直到进程退出/主动释放);
抢不到锁(已有其它 worker 持有)或打开失败 -> 返回 None。
进程异常退出时 OS 会自动释放该锁。
"""
try:
lock_path.parent.mkdir(parents=True, exist_ok=True)
handle = open(lock_path, "a+")
except OSError:
logger.exception("Curator scheduler: 打开锁文件失败")
return None
try:
if sys.platform == "win32":
import msvcrt
handle.seek(0)
msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1)
else:
import fcntl
fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
except OSError:
handle.close()
return None
return handle
def _release_singleton_lock(handle) -> None:
"""释放并关闭 _acquire_singleton_lock 拿到的锁文件句柄。"""
try:
if sys.platform == "win32":
import msvcrt
handle.seek(0)
msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
else:
import fcntl
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
except OSError:
pass
finally:
try:
handle.close()
except OSError:
pass
async def _curator_scheduler_loop() -> None:
"""常驻后台任务:每隔 _CURATOR_CHECK_INTERVAL_SECONDS 检查一次需不需要进化。
多 worker 部署下用文件锁做 leader 选举 —— 只有抢到锁的那个 worker 进程真正
运行调度,其余 worker 直接退出,避免 curator 被重复触发、重复调用 LLM。
被 cancel 时 CancelledError 会穿过 ``except Exception``(它是 BaseException
子类,不会被捕获),协程随之干净退出并在 finally 里释放锁。
"""
from deerflow.config.runtime_paths import runtime_home
from deerflow.skills.curator import maybe_run_curator
lock_handle = _acquire_singleton_lock(runtime_home() / ".curator_scheduler.lock")
if lock_handle is None:
logger.info("Curator scheduler: 另一个 worker 已持有调度锁,本 worker 跳过")
return
logger.info("Curator scheduler: 取得调度锁,本 worker 负责 curator 调度")
try:
# 启动后延迟一会儿再首次检查,避开启动高峰
await asyncio.sleep(_CURATOR_STARTUP_DELAY_SECONDS)
while True:
try:
await maybe_run_curator()
except Exception:
logger.exception("Curator scheduler tick failed (non-fatal)")
await asyncio.sleep(_CURATOR_CHECK_INTERVAL_SECONDS)
finally:
_release_singleton_lock(lock_handle)
@asynccontextmanager
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
"""Application lifespan handler."""
@@ -302,12 +390,12 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
except Exception:
logger.exception("Skill ownership migration failed (non-fatal)")
# Kick off skill curator in background if the run interval has elapsed
# 启动 skill curator 调度器:常驻后台任务,每小时检查一次需不需要进化
# (per-user interval_hours 仍是真正决定要不要跑进化的闸)。
try:
from deerflow.skills.curator import maybe_run_curator
asyncio.create_task(maybe_run_curator())
app.state.curator_scheduler_task = asyncio.create_task(_curator_scheduler_loop())
except Exception:
logger.exception("Skill curator startup task failed (non-fatal)")
logger.exception("Skill curator scheduler failed to start (non-fatal)")
# Start IM channel service if any channels are configured
try:
@@ -335,6 +423,17 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
yield
# 停止 curator 调度器
curator_task = getattr(app.state, "curator_scheduler_task", None)
if curator_task is not None:
curator_task.cancel()
try:
await curator_task
except asyncio.CancelledError:
pass
except Exception:
logger.exception("Curator scheduler shutdown error")
try:
scheduler_service = getattr(app.state, "scheduled_task_service", None)
if scheduler_service is not None:
@@ -3,9 +3,50 @@ from pydantic import BaseModel, Field
from app.gateway.deps import get_config
from deerflow.config.app_config import AppConfig
from deerflow.config.model_config import ModelConfig
router = APIRouter(prefix="/api", tags=["models"])
_PROVIDER_KEYWORDS: list[tuple[str, str]] = [
("langchain_openai", "openai"),
("langchain_anthropic", "anthropic"),
("langchain_google", "google"),
("langchain_groq", "groq"),
("langchain_mistralai", "mistral"),
("langchain_cohere", "cohere"),
("ollama", "ollama"),
("vllm", "vllm"),
]
_LOCAL_PROVIDERS = {"ollama", "vllm"}
def _derive_provider(use: str) -> str | None:
use_lower = use.lower()
for keyword, provider in _PROVIDER_KEYWORDS:
if keyword in use_lower:
return provider
return None
def _build_model_response(model: ModelConfig) -> "ModelResponse":
extra = model.model_extra or {}
provider = str(extra["provider"]) if "provider" in extra else _derive_provider(model.use)
is_local = bool(extra["is_local"]) if "is_local" in extra else (provider in _LOCAL_PROVIDERS)
supports_tool_calling = bool(extra["supports_tool_calling"]) if "supports_tool_calling" in extra else True
return ModelResponse(
name=model.name,
model=model.model,
display_name=model.display_name,
description=model.description,
supports_thinking=model.supports_thinking,
supports_reasoning_effort=model.supports_reasoning_effort,
supports_vision=model.supports_vision,
supports_tool_calling=supports_tool_calling,
is_local=is_local,
provider=provider,
)
class ModelResponse(BaseModel):
"""Response model for model information."""
@@ -16,6 +57,10 @@ class ModelResponse(BaseModel):
description: str | None = Field(None, description="Model description")
supports_thinking: bool = Field(default=False, description="Whether model supports thinking mode")
supports_reasoning_effort: bool = Field(default=False, description="Whether model supports reasoning effort")
supports_vision: bool = Field(default=False, description="Whether model supports image inputs")
supports_tool_calling: bool = Field(default=True, description="Whether model supports tool/function calling")
is_local: bool = Field(default=False, description="Whether this is a locally-hosted model")
provider: str | None = Field(None, description="Model provider name (openai, anthropic, ollama, etc.)")
class TokenUsageResponse(BaseModel):
@@ -73,17 +118,7 @@ async def list_models(config: AppConfig = Depends(get_config)) -> ModelsListResp
}
```
"""
models = [
ModelResponse(
name=model.name,
model=model.model,
display_name=model.display_name,
description=model.description,
supports_thinking=model.supports_thinking,
supports_reasoning_effort=model.supports_reasoning_effort,
)
for model in config.models
]
models = [_build_model_response(model) for model in config.models]
return ModelsListResponse(
models=models,
token_usage=TokenUsageResponse(enabled=config.token_usage.enabled),
@@ -122,11 +157,4 @@ async def get_model(model_name: str, config: AppConfig = Depends(get_config)) ->
if model is None:
raise HTTPException(status_code=404, detail=f"Model '{model_name}' not found")
return ModelResponse(
name=model.name,
model=model.model,
display_name=model.display_name,
description=model.description,
supports_thinking=model.supports_thinking,
supports_reasoning_effort=model.supports_reasoning_effort,
)
return _build_model_response(model)
@@ -1045,12 +1045,21 @@ async def curator_status(request: Request) -> dict:
@router.post("/curator/run", summary="Trigger Curator Run")
async def run_curator(request: Request) -> dict:
"""Manually trigger a curator review pass for the authenticated user
(bypasses interval check, runs in a background thread)."""
(bypasses the interval check, runs in a background thread).
Pre-checks the master switch / curator switch / run lock so the response
accurately reflects whether a run actually started — instead of always
reporting success. Returns ``{"triggered": bool, "reason": str | None}``.
"""
try:
import threading
from deerflow.skills.curator import _curator_thread_target
from deerflow.skills.curator import _curator_thread_target, manual_trigger_blocked_reason
user_id = _current_user_id(request)
reason = manual_trigger_blocked_reason(user_id)
if reason is not None:
return {"triggered": False, "reason": reason}
thread = threading.Thread(
target=_curator_thread_target,
args=(user_id,),
@@ -1058,7 +1067,7 @@ async def run_curator(request: Request) -> dict:
name=f"skill-curator-manual-{user_id}",
)
thread.start()
return {"triggered": True, "message": "Curator run started in background"}
return {"triggered": True, "reason": None}
except Exception as e:
logger.error("Failed to trigger curator: %s", e, exc_info=True)
raise HTTPException(status_code=500, detail=f"Failed to trigger curator: {str(e)}")
@@ -1067,17 +1076,19 @@ async def run_curator(request: Request) -> dict:
class CuratorSubConfig(BaseModel):
"""Nested curator configuration mirroring CuratorConfig."""
enabled: bool = Field(default=True, description="Whether the curator runs automatically.")
# 与 CuratorConfig.enabled 保持一致:默认关闭,需显式 opt-in。
enabled: bool = Field(default=False, description="Whether the curator runs automatically.")
interval_hours: int = Field(default=168, ge=1, description="Hours between curator runs.")
stale_after_days: int = Field(default=30, ge=1, description="Days of inactivity before a skill is marked stale.")
archive_after_days: int = Field(default=90, ge=1, description="Days of inactivity before a stale skill is archived.")
review_timeout_seconds: int = Field(default=600, ge=1, description="Overall timeout (seconds) for the LLM review pass.")
model_name: str | None = Field(default=None, description="Model name for the curator LLM review pass; null = primary model.")
class CuratorConfigResponse(BaseModel):
"""Response model exposing the effective skill_evolution config."""
enabled: bool
moderation_model_name: str | None
curator: CuratorSubConfig
@@ -1089,7 +1100,6 @@ class CuratorConfigRequest(BaseModel):
"""
enabled: bool = Field(..., description="Whether the agent can create and modify skills under skills/custom.")
moderation_model_name: str | None = Field(default=None, description="Optional model name for skill security moderation.")
curator: CuratorSubConfig
@@ -1113,12 +1123,13 @@ def _build_curator_config_response(evo) -> CuratorConfigResponse:
"""
return CuratorConfigResponse(
enabled=evo.enabled,
moderation_model_name=evo.moderation_model_name,
curator=CuratorSubConfig(
enabled=evo.curator.enabled,
interval_hours=evo.curator.interval_hours,
stale_after_days=evo.curator.stale_after_days,
archive_after_days=evo.curator.archive_after_days,
review_timeout_seconds=evo.curator.review_timeout_seconds,
model_name=evo.curator.model_name,
),
)
@@ -98,6 +98,10 @@ def normalize_input(raw_input: dict[str, Any] | None) -> dict[str, Any]:
_DEFAULT_ASSISTANT_ID = "lead_agent"
# Built-in LangGraph graph IDs registered in langgraph.json.
# These are NOT stored in the agents DB and must bypass the custom-agent access check.
_BUILTIN_GRAPH_IDS: frozenset[str] = frozenset({"lead_agent", "canvas_agent"})
def _prepend_prompt_prefix(graph_input: dict[str, Any], prefix: str) -> dict[str, Any]:
"""Return ``graph_input`` with the prefix *tagged* onto its first HumanMessage.
@@ -180,12 +184,15 @@ def merge_run_context_overrides(config: dict[str, Any], context: Mapping[str, An
def resolve_agent_factory(assistant_id: str | None):
"""Resolve the agent factory callable from config.
Custom agents are implemented as ``lead_agent`` + an ``agent_id``
injected into ``configurable`` or ``context`` — see
:func:`build_run_config`. All ``assistant_id`` values therefore map to the
same factory; the routing happens inside ``make_lead_agent`` when it reads
``cfg["agent_id"]``.
Built-in graphs (``lead_agent``, ``canvas_agent``) are dispatched directly
to their own factory. Custom agents share the ``lead_agent`` factory with
``agent_id`` injected into configurable/context by :func:`build_run_config`.
"""
if assistant_id == "canvas_agent":
from deerflow.agents.canvas_agent.agent import make_canvas_agent
return make_canvas_agent
from deerflow.agents.lead_agent.agent import make_lead_agent
return make_lead_agent
@@ -244,8 +251,9 @@ def build_run_config(
config["configurable"] = {"thread_id": thread_id}
# Inject custom agent id when the caller specified a non-default assistant.
# Honour an explicit agent_id in the active runtime options container.
if assistant_id and assistant_id != _DEFAULT_ASSISTANT_ID:
# Built-in graphs (lead_agent, canvas_agent) are dispatched via their own
# factory and must NOT receive an agent_id override.
if assistant_id and assistant_id not in _BUILTIN_GRAPH_IDS:
normalized = assistant_id.strip()
if not normalized or not re.fullmatch(r"[A-Za-z0-9_-]+", normalized):
raise ValueError(f"Invalid assistant_id {assistant_id!r}: must contain only letters, digits, hyphens, and underscores.")
@@ -265,7 +273,7 @@ def build_run_config(
def _collect_requested_agent_ids(body: Any) -> set[str]:
"""Collect agent ids from assistant_id/config/context for access checks."""
agent_ids: set[str] = set()
if body.assistant_id and body.assistant_id != _DEFAULT_ASSISTANT_ID:
if body.assistant_id and body.assistant_id not in _BUILTIN_GRAPH_IDS:
agent_ids.add(str(body.assistant_id))
def collect(container: Mapping[str, Any] | None) -> None: