Merge branch 'master' of http://192.168.0.200/knowledge-base-bj/deerflow-server into ji
# Conflicts: # frontend-web/src/components/workspace/chats/chat-box.tsx # offline-backend-20260512/backend/app/gateway/routers/agents.py
This commit is contained in:
commit
6792b4f4c4
200 files changed
+78520
-392
No files matched your search
@@ -5,6 +5,7 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import logging
|
||||
import shutil
|
||||
import time
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
@@ -207,6 +208,11 @@ async def _sync_legacy_agents(store: AgentStore) -> None:
|
||||
|
||||
This preserves old deployments where agents already existed under
|
||||
``.deer-flow/agents/{name}`` before the agents table was introduced.
|
||||
|
||||
Heavy: scans the agents directory + parses each ``config.yaml`` + issues a
|
||||
SELECT-then-INSERT/UPDATE per agent. Call once at startup via the lifespan,
|
||||
and rely on :func:`_maybe_sync_legacy_agents` (TTL-gated) inside request
|
||||
handlers as a safety net for environments that bypassed the lifespan hook.
|
||||
"""
|
||||
try:
|
||||
for agent_cfg in list_custom_agents():
|
||||
@@ -225,6 +231,44 @@ async def _sync_legacy_agents(store: AgentStore) -> None:
|
||||
logger.exception("Failed to sync legacy folder-backed agents")
|
||||
|
||||
|
||||
# Throttle the per-request legacy-sync fallback. The real sync runs once at
|
||||
# startup (lifespan); request handlers used to call ``_sync_legacy_agents`` on
|
||||
# every hit, which scanned the filesystem + did N DB upserts per agent each
|
||||
# time. After the first successful sync we skip the work for ``_LEGACY_SYNC_TTL_SECONDS``
|
||||
# — long enough to be effectively "startup-only" in steady state, short enough
|
||||
# that a freshly dropped agent directory gets picked up on the next bucket.
|
||||
_LEGACY_SYNC_TTL_SECONDS = 3600.0
|
||||
_legacy_sync_state: dict[str, Any] = {"last_at": 0.0, "lock": None}
|
||||
|
||||
|
||||
async def _maybe_sync_legacy_agents(store: AgentStore) -> None:
|
||||
"""TTL-gated wrapper around :func:`_sync_legacy_agents`.
|
||||
|
||||
Becomes a no-op in steady state — the heavy lifting happens in the lifespan
|
||||
hook. Inside the TTL window we don't even take the lock; outside it we
|
||||
serialize callers so a thundering herd doesn't fan out into N filesystem
|
||||
scans + N*M DB queries.
|
||||
"""
|
||||
now = time.monotonic()
|
||||
if now - float(_legacy_sync_state["last_at"]) < _LEGACY_SYNC_TTL_SECONDS:
|
||||
return
|
||||
lock = _legacy_sync_state["lock"]
|
||||
if lock is None:
|
||||
lock = asyncio.Lock()
|
||||
_legacy_sync_state["lock"] = lock
|
||||
async with lock:
|
||||
# Re-check under the lock — a sibling coroutine may have just refreshed.
|
||||
if time.monotonic() - float(_legacy_sync_state["last_at"]) < _LEGACY_SYNC_TTL_SECONDS:
|
||||
return
|
||||
await _sync_legacy_agents(store)
|
||||
_legacy_sync_state["last_at"] = time.monotonic()
|
||||
|
||||
|
||||
def _mark_legacy_sync_fresh() -> None:
|
||||
"""Called by the lifespan hook after running :func:`_sync_legacy_agents`."""
|
||||
_legacy_sync_state["last_at"] = time.monotonic()
|
||||
|
||||
|
||||
def _ensure_agent_dir_from_record(record: dict[str, Any]) -> None:
|
||||
"""Lazily restore an agent's on-disk artifacts from its DB record.
|
||||
|
||||
@@ -262,14 +306,29 @@ def _record_to_response(
|
||||
record: dict[str, Any],
|
||||
*,
|
||||
include_soul: bool = False,
|
||||
include_extras: bool = True,
|
||||
) -> AgentResponse:
|
||||
_ensure_agent_dir_from_record(record)
|
||||
"""Convert a DB row to an API response.
|
||||
|
||||
Args:
|
||||
include_soul: Read SOUL.md from disk and embed in ``soul``. Set False
|
||||
in list endpoints — the frontend (gallery / recommend / selector)
|
||||
never reads ``soul``, only the edit dialog does. Skipping this
|
||||
avoids one synchronous file read per agent.
|
||||
include_extras: Self-heal a missing agent dir and read deprecated
|
||||
``model`` / ``tool_groups`` / ``skills`` from ``config.yaml``. Set
|
||||
False in list endpoints to skip a stat + yaml parse per agent —
|
||||
those fields are unused by current frontends and the dir restore
|
||||
can happen lazily when the agent is actually opened.
|
||||
"""
|
||||
soul: str | None = None
|
||||
extras: dict[str, Any] = {"skills": None, "model": None, "tool_groups": None}
|
||||
if include_extras:
|
||||
_ensure_agent_dir_from_record(record)
|
||||
extras = _load_config_extras(record["id"])
|
||||
if include_soul:
|
||||
soul = load_agent_soul(record["id"]) or ""
|
||||
|
||||
extras = _load_config_extras(record["id"])
|
||||
|
||||
return AgentResponse(
|
||||
id=record["id"],
|
||||
name=record["name"],
|
||||
@@ -288,6 +347,25 @@ def _record_to_response(
|
||||
)
|
||||
|
||||
|
||||
async def _record_to_response_async(
|
||||
record: dict[str, Any],
|
||||
*,
|
||||
include_soul: bool = False,
|
||||
include_extras: bool = True,
|
||||
) -> AgentResponse:
|
||||
"""Async wrapper: when we actually touch disk, hop to a worker thread so
|
||||
we don't block the asyncio loop. List endpoints (no soul, no extras) skip
|
||||
this overhead and stay on the calling task."""
|
||||
if not include_soul and not include_extras:
|
||||
return _record_to_response(record, include_soul=False, include_extras=False)
|
||||
return await asyncio.to_thread(
|
||||
_record_to_response,
|
||||
record,
|
||||
include_soul=include_soul,
|
||||
include_extras=include_extras,
|
||||
)
|
||||
|
||||
|
||||
async def _get_owned_or_raise(store: AgentStore, agent_id: str, user_id: str) -> dict[str, Any]:
|
||||
owned = await store.get_owned(agent_id, user_id)
|
||||
if owned is not None:
|
||||
@@ -349,7 +427,7 @@ async def list_agents(
|
||||
try:
|
||||
store = get_agent_store(request)
|
||||
tag_store: TagStore = get_tag_store(request)
|
||||
await _sync_legacy_agents(store)
|
||||
await _maybe_sync_legacy_agents(store)
|
||||
user_id = _current_user_id(request)
|
||||
if admin:
|
||||
if not _is_admin_user(request):
|
||||
@@ -357,7 +435,14 @@ async def list_agents(
|
||||
agents = await store.list_all()
|
||||
else:
|
||||
agents = await store.list_visible(user_id)
|
||||
responses = [_record_to_response(a, include_soul=True) for a in agents]
|
||||
# List endpoint stays lean: skip per-agent SOUL.md read, config.yaml
|
||||
# parse, and dir self-heal — none of those feed UI fields the gallery /
|
||||
# recommend dialog / selector actually render. Detail endpoints (single
|
||||
# agent + create/update responses) still load them.
|
||||
responses = [
|
||||
_record_to_response(a, include_soul=False, include_extras=False)
|
||||
for a in agents
|
||||
]
|
||||
|
||||
assignments = await tag_store.list_assignments("agent", [r.id for r in responses])
|
||||
favorite_ids = await store.list_favorite_ids(user_id)
|
||||
@@ -403,11 +488,18 @@ async def get_agent_selector(request: Request) -> AgentSelectorResponse:
|
||||
_require_agents_api_enabled()
|
||||
try:
|
||||
store = get_agent_store(request)
|
||||
await _sync_legacy_agents(store)
|
||||
await _maybe_sync_legacy_agents(store)
|
||||
|
||||
user_id = _current_user_id(request)
|
||||
records = await store.list_featured(user_id)
|
||||
return AgentSelectorResponse(agents=[_record_to_response(r) for r in records])
|
||||
# Selector dropdown only renders name/description/featured_order — same
|
||||
# rationale as list_agents: skip soul + extras + self-heal.
|
||||
return AgentSelectorResponse(
|
||||
agents=[
|
||||
_record_to_response(r, include_soul=False, include_extras=False)
|
||||
for r in records
|
||||
]
|
||||
)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
@@ -427,12 +519,15 @@ async def get_agent(agent_id: str, request: Request) -> AgentResponse:
|
||||
|
||||
try:
|
||||
store = get_agent_store(request)
|
||||
await _sync_legacy_agents(store)
|
||||
await _maybe_sync_legacy_agents(store)
|
||||
user_id = _current_user_id(request)
|
||||
record = await store.get_visible(agent_id, user_id)
|
||||
if record is None:
|
||||
raise HTTPException(status_code=404, detail=f"Agent '{agent_id}' not found")
|
||||
response = _record_to_response(record, include_soul=True)
|
||||
# Detail view legitimately reads SOUL.md + config.yaml — hop to a
|
||||
# worker thread so the asyncio loop stays responsive under concurrent
|
||||
# detail fetches.
|
||||
response = await _record_to_response_async(record, include_soul=True, include_extras=True)
|
||||
favorite_ids = await store.list_favorite_ids(user_id)
|
||||
response.is_favorite = agent_id in favorite_ids
|
||||
return response
|
||||
@@ -488,7 +583,7 @@ async def create_agent_endpoint(request: AgentCreateRequest, http_request: Reque
|
||||
}
|
||||
)
|
||||
logger.info("Created agent '%s' (%s) at %s", name, agent_id, agent_dir)
|
||||
return _record_to_response(record, include_soul=True)
|
||||
return await _record_to_response_async(record, include_soul=True, include_extras=True)
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
@@ -569,7 +664,7 @@ async def update_agent(agent_id: str, request: AgentUpdateRequest, http_request:
|
||||
raise HTTPException(status_code=404, detail=f"Agent '{agent_id}' not found")
|
||||
|
||||
logger.info("Updated agent '%s' (%s)", record["name"], agent_id)
|
||||
return _record_to_response(record, include_soul=True)
|
||||
return await _record_to_response_async(record, include_soul=True, include_extras=True)
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
@@ -744,7 +839,7 @@ async def set_agent_featured(
|
||||
|
||||
try:
|
||||
store = get_agent_store(request)
|
||||
await _sync_legacy_agents(store)
|
||||
await _maybe_sync_legacy_agents(store)
|
||||
record = await store.get_any(agent_id)
|
||||
if record is None:
|
||||
raise HTTPException(status_code=404, detail=f"Agent '{agent_id}' not found")
|
||||
@@ -756,7 +851,7 @@ async def set_agent_featured(
|
||||
if updated is None:
|
||||
raise HTTPException(status_code=404, detail=f"Agent '{agent_id}' not found")
|
||||
logger.info("Set agent '%s' featured_order=%s by admin", agent_id, body.order)
|
||||
return _record_to_response(updated, include_soul=True)
|
||||
return await _record_to_response_async(updated, include_soul=True, include_extras=True)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
|
||||
Reference in new issue
Block a user