Merge remote-tracking branch 'origin/master' into ji
This commit is contained in:
commit
4761677676
74 files changed
+9770
-2887
No files matched your search
@@ -1,6 +1,7 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from collections.abc import AsyncGenerator
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
@@ -268,6 +269,93 @@ async def _run_db_migrations() -> None:
|
||||
logger.exception("Database migration failed — schema may be out of date")
|
||||
|
||||
|
||||
# Skill curator 调度参数。
|
||||
# - 检查间隔:每隔这么久检查一次"需不需要进化";真正是否执行由 should_run_now() 判定。
|
||||
# - 启动延迟:启动后先等这么久再首次检查,避开启动高峰。
|
||||
_CURATOR_CHECK_INTERVAL_SECONDS = 3600 # 1 小时
|
||||
_CURATOR_STARTUP_DELAY_SECONDS = 300 # 5 分钟
|
||||
|
||||
|
||||
def _acquire_singleton_lock(lock_path):
|
||||
"""跨平台非阻塞独占文件锁,用于多 worker 下的 leader 选举。
|
||||
|
||||
成功 -> 返回持有锁的文件句柄(调用方须保持其存活,直到进程退出/主动释放);
|
||||
抢不到锁(已有其它 worker 持有)或打开失败 -> 返回 None。
|
||||
进程异常退出时 OS 会自动释放该锁。
|
||||
"""
|
||||
try:
|
||||
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
handle = open(lock_path, "a+")
|
||||
except OSError:
|
||||
logger.exception("Curator scheduler: 打开锁文件失败")
|
||||
return None
|
||||
try:
|
||||
if sys.platform == "win32":
|
||||
import msvcrt
|
||||
|
||||
handle.seek(0)
|
||||
msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1)
|
||||
else:
|
||||
import fcntl
|
||||
|
||||
fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except OSError:
|
||||
handle.close()
|
||||
return None
|
||||
return handle
|
||||
|
||||
|
||||
def _release_singleton_lock(handle) -> None:
|
||||
"""释放并关闭 _acquire_singleton_lock 拿到的锁文件句柄。"""
|
||||
try:
|
||||
if sys.platform == "win32":
|
||||
import msvcrt
|
||||
|
||||
handle.seek(0)
|
||||
msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
|
||||
else:
|
||||
import fcntl
|
||||
|
||||
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
|
||||
except OSError:
|
||||
pass
|
||||
finally:
|
||||
try:
|
||||
handle.close()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
async def _curator_scheduler_loop() -> None:
|
||||
"""常驻后台任务:每隔 _CURATOR_CHECK_INTERVAL_SECONDS 检查一次需不需要进化。
|
||||
|
||||
多 worker 部署下用文件锁做 leader 选举 —— 只有抢到锁的那个 worker 进程真正
|
||||
运行调度,其余 worker 直接退出,避免 curator 被重复触发、重复调用 LLM。
|
||||
被 cancel 时 CancelledError 会穿过 ``except Exception``(它是 BaseException
|
||||
子类,不会被捕获),协程随之干净退出并在 finally 里释放锁。
|
||||
"""
|
||||
from deerflow.config.runtime_paths import runtime_home
|
||||
from deerflow.skills.curator import maybe_run_curator
|
||||
|
||||
lock_handle = _acquire_singleton_lock(runtime_home() / ".curator_scheduler.lock")
|
||||
if lock_handle is None:
|
||||
logger.info("Curator scheduler: 另一个 worker 已持有调度锁,本 worker 跳过")
|
||||
return
|
||||
|
||||
logger.info("Curator scheduler: 取得调度锁,本 worker 负责 curator 调度")
|
||||
try:
|
||||
# 启动后延迟一会儿再首次检查,避开启动高峰
|
||||
await asyncio.sleep(_CURATOR_STARTUP_DELAY_SECONDS)
|
||||
while True:
|
||||
try:
|
||||
await maybe_run_curator()
|
||||
except Exception:
|
||||
logger.exception("Curator scheduler tick failed (non-fatal)")
|
||||
await asyncio.sleep(_CURATOR_CHECK_INTERVAL_SECONDS)
|
||||
finally:
|
||||
_release_singleton_lock(lock_handle)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
"""Application lifespan handler."""
|
||||
@@ -302,12 +390,12 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
except Exception:
|
||||
logger.exception("Skill ownership migration failed (non-fatal)")
|
||||
|
||||
# Kick off skill curator in background if the run interval has elapsed
|
||||
# 启动 skill curator 调度器:常驻后台任务,每小时检查一次需不需要进化
|
||||
# (per-user interval_hours 仍是真正决定要不要跑进化的闸)。
|
||||
try:
|
||||
from deerflow.skills.curator import maybe_run_curator
|
||||
asyncio.create_task(maybe_run_curator())
|
||||
app.state.curator_scheduler_task = asyncio.create_task(_curator_scheduler_loop())
|
||||
except Exception:
|
||||
logger.exception("Skill curator startup task failed (non-fatal)")
|
||||
logger.exception("Skill curator scheduler failed to start (non-fatal)")
|
||||
|
||||
# Start IM channel service if any channels are configured
|
||||
try:
|
||||
@@ -335,6 +423,17 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
|
||||
yield
|
||||
|
||||
# 停止 curator 调度器
|
||||
curator_task = getattr(app.state, "curator_scheduler_task", None)
|
||||
if curator_task is not None:
|
||||
curator_task.cancel()
|
||||
try:
|
||||
await curator_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception:
|
||||
logger.exception("Curator scheduler shutdown error")
|
||||
|
||||
try:
|
||||
scheduler_service = getattr(app.state, "scheduled_task_service", None)
|
||||
if scheduler_service is not None:
|
||||
|
||||
@@ -3,9 +3,50 @@ from pydantic import BaseModel, Field
|
||||
|
||||
from app.gateway.deps import get_config
|
||||
from deerflow.config.app_config import AppConfig
|
||||
from deerflow.config.model_config import ModelConfig
|
||||
|
||||
router = APIRouter(prefix="/api", tags=["models"])
|
||||
|
||||
_PROVIDER_KEYWORDS: list[tuple[str, str]] = [
|
||||
("langchain_openai", "openai"),
|
||||
("langchain_anthropic", "anthropic"),
|
||||
("langchain_google", "google"),
|
||||
("langchain_groq", "groq"),
|
||||
("langchain_mistralai", "mistral"),
|
||||
("langchain_cohere", "cohere"),
|
||||
("ollama", "ollama"),
|
||||
("vllm", "vllm"),
|
||||
]
|
||||
|
||||
_LOCAL_PROVIDERS = {"ollama", "vllm"}
|
||||
|
||||
|
||||
def _derive_provider(use: str) -> str | None:
|
||||
use_lower = use.lower()
|
||||
for keyword, provider in _PROVIDER_KEYWORDS:
|
||||
if keyword in use_lower:
|
||||
return provider
|
||||
return None
|
||||
|
||||
|
||||
def _build_model_response(model: ModelConfig) -> "ModelResponse":
|
||||
extra = model.model_extra or {}
|
||||
provider = str(extra["provider"]) if "provider" in extra else _derive_provider(model.use)
|
||||
is_local = bool(extra["is_local"]) if "is_local" in extra else (provider in _LOCAL_PROVIDERS)
|
||||
supports_tool_calling = bool(extra["supports_tool_calling"]) if "supports_tool_calling" in extra else True
|
||||
return ModelResponse(
|
||||
name=model.name,
|
||||
model=model.model,
|
||||
display_name=model.display_name,
|
||||
description=model.description,
|
||||
supports_thinking=model.supports_thinking,
|
||||
supports_reasoning_effort=model.supports_reasoning_effort,
|
||||
supports_vision=model.supports_vision,
|
||||
supports_tool_calling=supports_tool_calling,
|
||||
is_local=is_local,
|
||||
provider=provider,
|
||||
)
|
||||
|
||||
|
||||
class ModelResponse(BaseModel):
|
||||
"""Response model for model information."""
|
||||
@@ -16,6 +57,10 @@ class ModelResponse(BaseModel):
|
||||
description: str | None = Field(None, description="Model description")
|
||||
supports_thinking: bool = Field(default=False, description="Whether model supports thinking mode")
|
||||
supports_reasoning_effort: bool = Field(default=False, description="Whether model supports reasoning effort")
|
||||
supports_vision: bool = Field(default=False, description="Whether model supports image inputs")
|
||||
supports_tool_calling: bool = Field(default=True, description="Whether model supports tool/function calling")
|
||||
is_local: bool = Field(default=False, description="Whether this is a locally-hosted model")
|
||||
provider: str | None = Field(None, description="Model provider name (openai, anthropic, ollama, etc.)")
|
||||
|
||||
|
||||
class TokenUsageResponse(BaseModel):
|
||||
@@ -73,17 +118,7 @@ async def list_models(config: AppConfig = Depends(get_config)) -> ModelsListResp
|
||||
}
|
||||
```
|
||||
"""
|
||||
models = [
|
||||
ModelResponse(
|
||||
name=model.name,
|
||||
model=model.model,
|
||||
display_name=model.display_name,
|
||||
description=model.description,
|
||||
supports_thinking=model.supports_thinking,
|
||||
supports_reasoning_effort=model.supports_reasoning_effort,
|
||||
)
|
||||
for model in config.models
|
||||
]
|
||||
models = [_build_model_response(model) for model in config.models]
|
||||
return ModelsListResponse(
|
||||
models=models,
|
||||
token_usage=TokenUsageResponse(enabled=config.token_usage.enabled),
|
||||
@@ -122,11 +157,4 @@ async def get_model(model_name: str, config: AppConfig = Depends(get_config)) ->
|
||||
if model is None:
|
||||
raise HTTPException(status_code=404, detail=f"Model '{model_name}' not found")
|
||||
|
||||
return ModelResponse(
|
||||
name=model.name,
|
||||
model=model.model,
|
||||
display_name=model.display_name,
|
||||
description=model.description,
|
||||
supports_thinking=model.supports_thinking,
|
||||
supports_reasoning_effort=model.supports_reasoning_effort,
|
||||
)
|
||||
return _build_model_response(model)
|
||||
@@ -1045,12 +1045,21 @@ async def curator_status(request: Request) -> dict:
|
||||
@router.post("/curator/run", summary="Trigger Curator Run")
|
||||
async def run_curator(request: Request) -> dict:
|
||||
"""Manually trigger a curator review pass for the authenticated user
|
||||
(bypasses interval check, runs in a background thread)."""
|
||||
(bypasses the interval check, runs in a background thread).
|
||||
|
||||
Pre-checks the master switch / curator switch / run lock so the response
|
||||
accurately reflects whether a run actually started — instead of always
|
||||
reporting success. Returns ``{"triggered": bool, "reason": str | None}``.
|
||||
"""
|
||||
try:
|
||||
import threading
|
||||
from deerflow.skills.curator import _curator_thread_target
|
||||
|
||||
from deerflow.skills.curator import _curator_thread_target, manual_trigger_blocked_reason
|
||||
|
||||
user_id = _current_user_id(request)
|
||||
reason = manual_trigger_blocked_reason(user_id)
|
||||
if reason is not None:
|
||||
return {"triggered": False, "reason": reason}
|
||||
thread = threading.Thread(
|
||||
target=_curator_thread_target,
|
||||
args=(user_id,),
|
||||
@@ -1058,7 +1067,7 @@ async def run_curator(request: Request) -> dict:
|
||||
name=f"skill-curator-manual-{user_id}",
|
||||
)
|
||||
thread.start()
|
||||
return {"triggered": True, "message": "Curator run started in background"}
|
||||
return {"triggered": True, "reason": None}
|
||||
except Exception as e:
|
||||
logger.error("Failed to trigger curator: %s", e, exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=f"Failed to trigger curator: {str(e)}")
|
||||
@@ -1067,17 +1076,19 @@ async def run_curator(request: Request) -> dict:
|
||||
class CuratorSubConfig(BaseModel):
|
||||
"""Nested curator configuration mirroring CuratorConfig."""
|
||||
|
||||
enabled: bool = Field(default=True, description="Whether the curator runs automatically.")
|
||||
# 与 CuratorConfig.enabled 保持一致:默认关闭,需显式 opt-in。
|
||||
enabled: bool = Field(default=False, description="Whether the curator runs automatically.")
|
||||
interval_hours: int = Field(default=168, ge=1, description="Hours between curator runs.")
|
||||
stale_after_days: int = Field(default=30, ge=1, description="Days of inactivity before a skill is marked stale.")
|
||||
archive_after_days: int = Field(default=90, ge=1, description="Days of inactivity before a stale skill is archived.")
|
||||
review_timeout_seconds: int = Field(default=600, ge=1, description="Overall timeout (seconds) for the LLM review pass.")
|
||||
model_name: str | None = Field(default=None, description="Model name for the curator LLM review pass; null = primary model.")
|
||||
|
||||
|
||||
class CuratorConfigResponse(BaseModel):
|
||||
"""Response model exposing the effective skill_evolution config."""
|
||||
|
||||
enabled: bool
|
||||
moderation_model_name: str | None
|
||||
curator: CuratorSubConfig
|
||||
|
||||
|
||||
@@ -1089,7 +1100,6 @@ class CuratorConfigRequest(BaseModel):
|
||||
"""
|
||||
|
||||
enabled: bool = Field(..., description="Whether the agent can create and modify skills under skills/custom.")
|
||||
moderation_model_name: str | None = Field(default=None, description="Optional model name for skill security moderation.")
|
||||
curator: CuratorSubConfig
|
||||
|
||||
|
||||
@@ -1113,12 +1123,13 @@ def _build_curator_config_response(evo) -> CuratorConfigResponse:
|
||||
"""
|
||||
return CuratorConfigResponse(
|
||||
enabled=evo.enabled,
|
||||
moderation_model_name=evo.moderation_model_name,
|
||||
curator=CuratorSubConfig(
|
||||
enabled=evo.curator.enabled,
|
||||
interval_hours=evo.curator.interval_hours,
|
||||
stale_after_days=evo.curator.stale_after_days,
|
||||
archive_after_days=evo.curator.archive_after_days,
|
||||
review_timeout_seconds=evo.curator.review_timeout_seconds,
|
||||
model_name=evo.curator.model_name,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -98,6 +98,10 @@ def normalize_input(raw_input: dict[str, Any] | None) -> dict[str, Any]:
|
||||
|
||||
_DEFAULT_ASSISTANT_ID = "lead_agent"
|
||||
|
||||
# Built-in LangGraph graph IDs registered in langgraph.json.
|
||||
# These are NOT stored in the agents DB and must bypass the custom-agent access check.
|
||||
_BUILTIN_GRAPH_IDS: frozenset[str] = frozenset({"lead_agent", "canvas_agent"})
|
||||
|
||||
|
||||
def _prepend_prompt_prefix(graph_input: dict[str, Any], prefix: str) -> dict[str, Any]:
|
||||
"""Return ``graph_input`` with the prefix *tagged* onto its first HumanMessage.
|
||||
@@ -180,12 +184,15 @@ def merge_run_context_overrides(config: dict[str, Any], context: Mapping[str, An
|
||||
def resolve_agent_factory(assistant_id: str | None):
|
||||
"""Resolve the agent factory callable from config.
|
||||
|
||||
Custom agents are implemented as ``lead_agent`` + an ``agent_id``
|
||||
injected into ``configurable`` or ``context`` — see
|
||||
:func:`build_run_config`. All ``assistant_id`` values therefore map to the
|
||||
same factory; the routing happens inside ``make_lead_agent`` when it reads
|
||||
``cfg["agent_id"]``.
|
||||
Built-in graphs (``lead_agent``, ``canvas_agent``) are dispatched directly
|
||||
to their own factory. Custom agents share the ``lead_agent`` factory with
|
||||
``agent_id`` injected into configurable/context by :func:`build_run_config`.
|
||||
"""
|
||||
if assistant_id == "canvas_agent":
|
||||
from deerflow.agents.canvas_agent.agent import make_canvas_agent
|
||||
|
||||
return make_canvas_agent
|
||||
|
||||
from deerflow.agents.lead_agent.agent import make_lead_agent
|
||||
|
||||
return make_lead_agent
|
||||
@@ -244,8 +251,9 @@ def build_run_config(
|
||||
config["configurable"] = {"thread_id": thread_id}
|
||||
|
||||
# Inject custom agent id when the caller specified a non-default assistant.
|
||||
# Honour an explicit agent_id in the active runtime options container.
|
||||
if assistant_id and assistant_id != _DEFAULT_ASSISTANT_ID:
|
||||
# Built-in graphs (lead_agent, canvas_agent) are dispatched via their own
|
||||
# factory and must NOT receive an agent_id override.
|
||||
if assistant_id and assistant_id not in _BUILTIN_GRAPH_IDS:
|
||||
normalized = assistant_id.strip()
|
||||
if not normalized or not re.fullmatch(r"[A-Za-z0-9_-]+", normalized):
|
||||
raise ValueError(f"Invalid assistant_id {assistant_id!r}: must contain only letters, digits, hyphens, and underscores.")
|
||||
@@ -265,7 +273,7 @@ def build_run_config(
|
||||
def _collect_requested_agent_ids(body: Any) -> set[str]:
|
||||
"""Collect agent ids from assistant_id/config/context for access checks."""
|
||||
agent_ids: set[str] = set()
|
||||
if body.assistant_id and body.assistant_id != _DEFAULT_ASSISTANT_ID:
|
||||
if body.assistant_id and body.assistant_id not in _BUILTIN_GRAPH_IDS:
|
||||
agent_ids.add(str(body.assistant_id))
|
||||
|
||||
def collect(container: Mapping[str, Any] | None) -> None:
|
||||
|
||||
Reference in new issue
Block a user