Merge origin/master into xuhd

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

offline-backend-20260512/backend/app/gateway/routers/agents.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 import asyncio
 import logging
 import shutil
+import time
 from datetime import datetime
 from typing import Any
 from uuid import uuid4
@@ -207,6 +208,11 @@ async def _sync_legacy_agents(store: AgentStore) -> None:
 
     This preserves old deployments where agents already existed under
     ``.deer-flow/agents/{name}`` before the agents table was introduced.
+
+    Heavy: scans the agents directory + parses each ``config.yaml`` + issues a
+    SELECT-then-INSERT/UPDATE per agent. Call once at startup via the lifespan,
+    and rely on :func:`_maybe_sync_legacy_agents` (TTL-gated) inside request
+    handlers as a safety net for environments that bypassed the lifespan hook.
     """
     try:
         for agent_cfg in list_custom_agents():
@@ -225,6 +231,44 @@ async def _sync_legacy_agents(store: AgentStore) -> None:
         logger.exception("Failed to sync legacy folder-backed agents")
 
 
+# Throttle the per-request legacy-sync fallback. The real sync runs once at
+# startup (lifespan); request handlers used to call ``_sync_legacy_agents`` on
+# every hit, which scanned the filesystem + did N DB upserts per agent each
+# time. After the first successful sync we skip the work for ``_LEGACY_SYNC_TTL_SECONDS``
+# — long enough to be effectively "startup-only" in steady state, short enough
+# that a freshly dropped agent directory gets picked up on the next bucket.
+_LEGACY_SYNC_TTL_SECONDS = 3600.0
+_legacy_sync_state: dict[str, Any] = {"last_at": 0.0, "lock": None}
+
+
+async def _maybe_sync_legacy_agents(store: AgentStore) -> None:
+    """TTL-gated wrapper around :func:`_sync_legacy_agents`.
+
+    Becomes a no-op in steady state — the heavy lifting happens in the lifespan
+    hook. Inside the TTL window we don't even take the lock; outside it we
+    serialize callers so a thundering herd doesn't fan out into N filesystem
+    scans + N*M DB queries.
+    """
+    now = time.monotonic()
+    if now - float(_legacy_sync_state["last_at"]) < _LEGACY_SYNC_TTL_SECONDS:
+        return
+    lock = _legacy_sync_state["lock"]
+    if lock is None:
+        lock = asyncio.Lock()
+        _legacy_sync_state["lock"] = lock
+    async with lock:
+        # Re-check under the lock — a sibling coroutine may have just refreshed.
+        if time.monotonic() - float(_legacy_sync_state["last_at"]) < _LEGACY_SYNC_TTL_SECONDS:
+            return
+        await _sync_legacy_agents(store)
+        _legacy_sync_state["last_at"] = time.monotonic()
+
+
+def _mark_legacy_sync_fresh() -> None:
+    """Called by the lifespan hook after running :func:`_sync_legacy_agents`."""
+    _legacy_sync_state["last_at"] = time.monotonic()
+
+
 def _ensure_agent_dir_from_record(record: dict[str, Any]) -> None:
     """Lazily restore an agent's on-disk artifacts from its DB record.
 
@@ -262,14 +306,29 @@ def _record_to_response(
     record: dict[str, Any],
     *,
     include_soul: bool = False,
+    include_extras: bool = True,
 ) -> AgentResponse:
-    _ensure_agent_dir_from_record(record)
+    """Convert a DB row to an API response.
+
+    Args:
+        include_soul: Read SOUL.md from disk and embed in ``soul``. Set False
+            in list endpoints — the frontend (gallery / recommend / selector)
+            never reads ``soul``, only the edit dialog does. Skipping this
+            avoids one synchronous file read per agent.
+        include_extras: Self-heal a missing agent dir and read deprecated
+            ``model`` / ``tool_groups`` / ``skills`` from ``config.yaml``. Set
+            False in list endpoints to skip a stat + yaml parse per agent —
+            those fields are unused by current frontends and the dir restore
+            can happen lazily when the agent is actually opened.
+    """
     soul: str | None = None
+    extras: dict[str, Any] = {"skills": None, "model": None, "tool_groups": None}
+    if include_extras:
+        _ensure_agent_dir_from_record(record)
+        extras = _load_config_extras(record["id"])
     if include_soul:
         soul = load_agent_soul(record["id"]) or ""
 
-    extras = _load_config_extras(record["id"])
-
     return AgentResponse(
         id=record["id"],
         name=record["name"],
@@ -288,6 +347,25 @@ def _record_to_response(
     )
 
 
+async def _record_to_response_async(
+    record: dict[str, Any],
+    *,
+    include_soul: bool = False,
+    include_extras: bool = True,
+) -> AgentResponse:
+    """Async wrapper: when we actually touch disk, hop to a worker thread so
+    we don't block the asyncio loop. List endpoints (no soul, no extras) skip
+    this overhead and stay on the calling task."""
+    if not include_soul and not include_extras:
+        return _record_to_response(record, include_soul=False, include_extras=False)
+    return await asyncio.to_thread(
+        _record_to_response,
+        record,
+        include_soul=include_soul,
+        include_extras=include_extras,
+    )
+
+
 async def _get_owned_or_raise(store: AgentStore, agent_id: str, user_id: str) -> dict[str, Any]:
     owned = await store.get_owned(agent_id, user_id)
     if owned is not None:
@@ -349,7 +427,7 @@ async def list_agents(
     try:
         store = get_agent_store(request)
         tag_store: TagStore = get_tag_store(request)
-        await _sync_legacy_agents(store)
+        await _maybe_sync_legacy_agents(store)
         user_id = _current_user_id(request)
         if admin:
             if not _is_admin_user(request):
@@ -357,7 +435,14 @@ async def list_agents(
             agents = await store.list_all()
         else:
             agents = await store.list_visible(user_id)
-        responses = [_record_to_response(a, include_soul=True) for a in agents]
+        # List endpoint stays lean: skip per-agent SOUL.md read, config.yaml
+        # parse, and dir self-heal — none of those feed UI fields the gallery /
+        # recommend dialog / selector actually render. Detail endpoints (single
+        # agent + create/update responses) still load them.
+        responses = [
+            _record_to_response(a, include_soul=False, include_extras=False)
+            for a in agents
+        ]
 
         assignments = await tag_store.list_assignments("agent", [r.id for r in responses])
         favorite_ids = await store.list_favorite_ids(user_id)
@@ -403,11 +488,18 @@ async def get_agent_selector(request: Request) -> AgentSelectorResponse:
     _require_agents_api_enabled()
     try:
         store = get_agent_store(request)
-        await _sync_legacy_agents(store)
+        await _maybe_sync_legacy_agents(store)
 
         user_id = _current_user_id(request)
         records = await store.list_featured(user_id)
-        return AgentSelectorResponse(agents=[_record_to_response(r) for r in records])
+        # Selector dropdown only renders name/description/featured_order — same
+        # rationale as list_agents: skip soul + extras + self-heal.
+        return AgentSelectorResponse(
+            agents=[
+                _record_to_response(r, include_soul=False, include_extras=False)
+                for r in records
+            ]
+        )
     except HTTPException:
         raise
     except Exception as e:
@@ -427,12 +519,15 @@ async def get_agent(agent_id: str, request: Request) -> AgentResponse:
 
     try:
         store = get_agent_store(request)
-        await _sync_legacy_agents(store)
+        await _maybe_sync_legacy_agents(store)
         user_id = _current_user_id(request)
         record = await store.get_visible(agent_id, user_id)
         if record is None:
             raise HTTPException(status_code=404, detail=f"Agent '{agent_id}' not found")
-        response = _record_to_response(record, include_soul=True)
+        # Detail view legitimately reads SOUL.md + config.yaml — hop to a
+        # worker thread so the asyncio loop stays responsive under concurrent
+        # detail fetches.
+        response = await _record_to_response_async(record, include_soul=True, include_extras=True)
         favorite_ids = await store.list_favorite_ids(user_id)
         response.is_favorite = agent_id in favorite_ids
         return response
@@ -488,7 +583,7 @@ async def create_agent_endpoint(request: AgentCreateRequest, http_request: Reque
             }
         )
         logger.info("Created agent '%s' (%s) at %s", name, agent_id, agent_dir)
-        return _record_to_response(record, include_soul=True)
+        return await _record_to_response_async(record, include_soul=True, include_extras=True)
 
     except HTTPException:
         raise
@@ -569,7 +664,7 @@ async def update_agent(agent_id: str, request: AgentUpdateRequest, http_request:
             raise HTTPException(status_code=404, detail=f"Agent '{agent_id}' not found")
 
         logger.info("Updated agent '%s' (%s)", record["name"], agent_id)
-        return _record_to_response(record, include_soul=True)
+        return await _record_to_response_async(record, include_soul=True, include_extras=True)
 
     except HTTPException:
         raise
@@ -744,7 +839,7 @@ async def set_agent_featured(
 
     try:
         store = get_agent_store(request)
-        await _sync_legacy_agents(store)
+        await _maybe_sync_legacy_agents(store)
         record = await store.get_any(agent_id)
         if record is None:
             raise HTTPException(status_code=404, detail=f"Agent '{agent_id}' not found")
@@ -756,7 +851,7 @@ async def set_agent_featured(
         if updated is None:
             raise HTTPException(status_code=404, detail=f"Agent '{agent_id}' not found")
         logger.info("Set agent '%s' featured_order=%s by admin", agent_id, body.order)
-        return _record_to_response(updated, include_soul=True)
+        return await _record_to_response_async(updated, include_soul=True, include_extras=True)
     except HTTPException:
         raise
     except Exception as e:

offline-backend-20260512/backend/config.yaml

@@ -58,16 +58,6 @@ models:
     temperature: 1.0
     supports_vision: true
     supports_thinking: true
-  - name: minimax-m2.5
-    display_name: MiniMax M2.5
-    use: langchain_openai:ChatOpenAI
-    model: MiniMax-M2.5
-    api_key: sk-cp-mrsAGtzDpUl0TpOBPVoiRG_pWF1kDDN9_mDAnONBs6HIPcNUexSKtSbx2QOxfKBvJT6Xw6hDTV6hdkJYFWfNA8UtHQ08TiGJiosdRoW6q1sGb6Cb7D85NHk
-    base_url: https://api.minimaxi.com/v1
-    max_tokens: 4096
-    temperature: 1.0
-    supports_vision: true
-    supports_thinking: false
 
 title:
   enabled: true

offline-backend-20260512/backend/packages/harness/deerflow/agents/lead_agent/agent.py

@@ -7,10 +7,10 @@ from langchain_core.runnables import RunnableConfig
 from deerflow.agents.lead_agent.prompt import apply_prompt_template
 from deerflow.agents.middlewares.clarification_middleware import ClarificationMiddleware
 from deerflow.agents.middlewares.loop_detection_middleware import LoopDetectionMiddleware
-from deerflow.agents.middlewares.memory_middleware import MemoryMiddleware
-from deerflow.agents.middlewares.prompt_prefix_middleware import PromptPrefixMiddleware
-from deerflow.agents.middlewares.rag_citation_middleware import RagCitationMiddleware
-from deerflow.agents.middlewares.setup_agent_approval_middleware import SetupAgentApprovalMiddleware
+from deerflow.agents.middlewares.memory_middleware import MemoryMiddleware
+from deerflow.agents.middlewares.prompt_prefix_middleware import PromptPrefixMiddleware
+from deerflow.agents.middlewares.rag_citation_middleware import RagCitationMiddleware
+from deerflow.agents.middlewares.setup_agent_approval_middleware import SetupAgentApprovalMiddleware
 from deerflow.agents.middlewares.subagent_limit_middleware import SubagentLimitMiddleware
 from deerflow.agents.middlewares.summarization_middleware import BeforeSummarizationHook, DeerFlowSummarizationMiddleware
 from deerflow.agents.middlewares.title_middleware import TitleMiddleware
@@ -253,16 +253,16 @@ def _build_middlewares(
 
     Returns:
         List of middleware instances.
-    """
-    resolved_app_config = app_config or get_app_config()
-    cfg = _get_runtime_config(config)
-    middlewares = build_lead_runtime_middlewares(app_config=resolved_app_config, lazy_init=True)
+    """
+    resolved_app_config = app_config or get_app_config()
+    cfg = _get_runtime_config(config)
+    middlewares = build_lead_runtime_middlewares(app_config=resolved_app_config, lazy_init=True)
 
-    # Inject the admin prompt prefix into the model request (kept out of the
-    # persisted/displayed message so the user never sees it).
-    middlewares.append(PromptPrefixMiddleware())
-    if cfg.get("rag_mode_enabled", False):
-        middlewares.append(RagCitationMiddleware())
+    # Inject the admin prompt prefix into the model request (kept out of the
+    # persisted/displayed message so the user never sees it).
+    middlewares.append(PromptPrefixMiddleware())
+    if cfg.get("rag_mode_enabled", False):
+        middlewares.append(RagCitationMiddleware())
 
     # Add summarization middleware if enabled
     summarization_middleware = _create_summarization_middleware(app_config=resolved_app_config)
@@ -270,7 +270,7 @@ def _build_middlewares(
         middlewares.append(summarization_middleware)
 
     # Add TodoList middleware if plan mode is enabled
-    is_plan_mode = cfg.get("is_plan_mode", False)
+    is_plan_mode = cfg.get("is_plan_mode", False)
     is_bootstrap = cfg.get("is_bootstrap", False)
     todo_list_middleware = _create_todo_list_middleware(is_plan_mode)
     if todo_list_middleware is not None:
@@ -374,6 +374,7 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
     max_concurrent_subagents = cfg.get("max_concurrent_subagents", 3)
     is_bootstrap = cfg.get("is_bootstrap", False)
     memory_injection_enabled = bool(cfg.get("memory_injection_enabled", True))
+    rag_mode_enabled = bool(cfg.get("rag_mode_enabled", False))
     writing_mode = bool(cfg.get("writing_mode", False))
     writing_artifact_path = cfg.get("writing_artifact_path") if writing_mode else None
     if not isinstance(writing_artifact_path, str) or not writing_artifact_path.strip():
@@ -464,6 +465,7 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
                 is_bootstrap=True,
                 app_config=resolved_app_config,
                 memory_injection_enabled=memory_injection_enabled,
+                rag_mode_enabled=rag_mode_enabled,
             ),
             state_schema=ThreadState,
         )
@@ -498,6 +500,7 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
             available_skills=(set(agent_config.skills or []) if agent_config else None),
             app_config=resolved_app_config,
             memory_injection_enabled=memory_injection_enabled,
+            rag_mode_enabled=rag_mode_enabled,
             is_notebook_mode=is_notebook_mode,
             notebook_source_ids=notebook_source_ids,
         ),

offline-backend-20260512/backend/packages/harness/deerflow/agents/lead_agent/prompt.py

@@ -667,17 +667,25 @@ def _get_memory_context(agent_name: str | None = None, *, app_config: AppConfig
 @lru_cache(maxsize=32)
 def _get_cached_skills_prompt_section(
     skill_signature: tuple[tuple[str, str, str, str], ...],
+    display_signature: tuple[tuple[str, str, str], ...],
     available_skills_key: tuple[str, ...] | None,
     container_base_path: str,
     skill_evolution_section: str,
 ) -> str:
+    display_by_skill = {name: (mode, custom_instruction) for name, mode, custom_instruction in display_signature}
     filtered = [(name, description, category, location) for name, description, category, location in skill_signature if available_skills_key is None or name in available_skills_key]
     skills_list = ""
     if filtered:
-        skill_items = "\n".join(
-            f"    <skill>\n        <name>{name}</name>\n        <description>{description} {_skill_mutability_label(category)}</description>\n        <location>{location}</location>\n    </skill>"
-            for name, description, category, location in filtered
-        )
+        items = []
+        for name, description, category, location in filtered:
+            display_mode, custom_instruction = display_by_skill.get(name, ("", ""))
+            display_hint = ""
+            if display_mode:
+                display_hint = f"\n        <display mode=\"{display_mode}\">{custom_instruction}</display>"
+            items.append(
+                f"    <skill>\n        <name>{name}</name>\n        <description>{description} {_skill_mutability_label(category)}</description>\n        <location>{location}</location>{display_hint}\n    </skill>"
+            )
+        skill_items = "\n".join(items)
         skills_list = f"<available_skills>\n{skill_items}\n</available_skills>"
 
     # Restricted mode: agent has an explicit skill allowlist.
@@ -693,6 +701,19 @@ def _get_cached_skills_prompt_section(
     else:
         scope_note = ""
 
+    display_pattern = (
+        """
+**Skill Result Display Pattern:**
+- If a skill has `<display mode="citation">`, use the skill's returned result order as reference numbers. Cite facts from the first result as `[[1]]`, the second as `[[2]]`, and so on. Use only numbers that correspond to returned results.
+- If a skill has `<display mode="list">`, summarize the returned items as a concise list and keep item order stable.
+- If a skill has `<display mode="table">`, summarize the result and output `<skill-display tool="SKILL_OR_TOOL_NAME" mode="table" />` where the frontend should render the real table. Do not hand-write the table rows yourself.
+- If a skill has `<display mode="card">`, summarize the result and output `<skill-display tool="SKILL_OR_TOOL_NAME" mode="card" />` where the frontend should render the cards.
+- If a skill has `<display mode="custom">`, follow the custom display instruction in the `<display>` tag when presenting the skill result.
+"""
+        if display_signature
+        else ""
+    )
+
     return f"""<skill_system>
 You have access to skills that provide optimized workflows for specific tasks. Each skill contains best practices, frameworks, and references to additional resources.
 {scope_note}
@@ -702,13 +723,19 @@ You have access to skills that provide optimized workflows for specific tasks. E
 3. The skill file contains references to external resources under the same folder
 4. Load referenced resources only when needed during execution
 5. Follow the skill's instructions precisely
+{display_pattern}
 {location_hint}{evolution_block}
 {skills_list}
 
 </skill_system>"""
 
 
-def get_skills_prompt_section(available_skills: set[str] | None = None, *, app_config: AppConfig | None = None) -> str:
+def get_skills_prompt_section(
+    available_skills: set[str] | None = None,
+    *,
+    app_config: AppConfig | None = None,
+    display_enabled: bool = False,
+) -> str:
     """Generate the skills prompt section with available skills list."""
     skills = _get_enabled_skills_for_config(app_config)
 
@@ -739,11 +766,30 @@ def get_skills_prompt_section(available_skills: set[str] | None = None, *, app_c
         return ""
 
     skill_signature = tuple((skill.name, skill.description, skill.category, skill.get_container_file_path(container_base_path)) for skill in skills)
+    display_signature: tuple[tuple[str, str, str], ...] = ()
+    if display_enabled:
+        try:
+            from deerflow.config.extensions_config import get_extensions_config
+
+            extensions = get_extensions_config()
+            display_signature = tuple(
+                (
+                    name,
+                    skill_config.display.mode,
+                    skill_config.display.custom_instruction,
+                )
+                for name, skill_config in sorted(extensions.skills.items())
+                if skill_config.display is not None
+                and skill_config.display.enabled
+                and skill_config.display.mode != "none"
+            )
+        except Exception:
+            logger.debug("Failed to load skill display configs for prompt injection", exc_info=True)
     available_key = tuple(sorted(available_skills)) if available_skills is not None else None
     if not skill_signature and available_key is not None:
         return ""
     skill_evolution_section = _build_skill_evolution_section(effective_skill_evolution_enabled)
-    return _get_cached_skills_prompt_section(skill_signature, available_key, container_base_path, skill_evolution_section)
+    return _get_cached_skills_prompt_section(skill_signature, display_signature, available_key, container_base_path, skill_evolution_section)
 
 
 def get_agent_soul(agent_id: str | None) -> str:
@@ -868,6 +914,7 @@ def apply_prompt_template(
     is_bootstrap: bool = False,
     app_config: AppConfig | None = None,
     memory_injection_enabled: bool = True,
+    rag_mode_enabled: bool = False,
     is_notebook_mode: bool = False,
     notebook_source_ids: list[int] | None = None,
 ) -> str:
@@ -902,7 +949,11 @@ def apply_prompt_template(
     bootstrap_section = _build_bootstrap_section() if is_bootstrap else ""
 
     # Get skills section
-    skills_section = get_skills_prompt_section(available_skills, app_config=app_config)
+    skills_section = get_skills_prompt_section(
+        available_skills,
+        app_config=app_config,
+        display_enabled=rag_mode_enabled,
+    )
 
     # Get deferred tools section (tool_search)
     deferred_tools_section = get_deferred_tools_prompt_section(app_config=app_config)
@@ -940,6 +991,21 @@ Writing mode is enabled for this run.
 
     # Build the offline-network policy block (empty unless DEER_FLOW_OFFLINE_MODE is set).
     network_policy_section = _build_network_policy_section()
+    rag_mode_section = (
+        """
+<rag_citation_mode>
+RAG/reference display mode is enabled.
+When a search, knowledge, skill, or MCP tool returns a result list:
+- If the skill has a display adapter, follow that adapter first.
+- For `citation`, cite returned items with `[[1]]`, `[[2]]`, etc. Only cite real returned items.
+- For `table`, do not hand-write the table. Summarize and place `<skill-display tool="SKILL_OR_TOOL_NAME" mode="table" />` where the table should appear. The frontend will render the real tool result as a stable table.
+- For `list` or `card`, summarize and place `<skill-display tool="SKILL_OR_TOOL_NAME" mode="list" />` or `<skill-display tool="SKILL_OR_TOOL_NAME" mode="card" />` where the structured result should appear.
+- If no display adapter exists, default to citation mode with numbered citations.
+</rag_citation_mode>
+"""
+        if rag_mode_enabled
+        else ""
+    )
 
     # Format the prompt with dynamic skills and memory
     prompt = SYSTEM_PROMPT_TEMPLATE.format(
@@ -957,7 +1023,7 @@ Writing mode is enabled for this run.
         acp_section=acp_and_mounts_section,
     )
 
-    prompt = prompt + f"\n<current_date>{datetime.now().strftime('%Y-%m-%d, %A')}</current_date>"
+    prompt = prompt + rag_mode_section + f"\n<current_date>{datetime.now().strftime('%Y-%m-%d, %A')}</current_date>"
 
     if is_notebook_mode:
         source_ids = notebook_source_ids or []