This commit is contained in:
fengxiangdebaozi committed 2026-06-24 19:48:15 +08:00
commit 3ea8e73389
61 files changed
+734 -429

No files matched your search

@@ -27,7 +27,7 @@ from app.gateway.routers.thread_runs import RunCreateRequest
from app.gateway.routers.threads import ThreadCreateRequest, create_thread
from app.gateway.routers.uploads import _make_file_sandbox_writable
from app.gateway.services import start_run
from deerflow.browser_context.tasks import claim_next_browser_task, complete_browser_task, prune_browser_tasks
from deerflow.browser_context.tasks import claim_next_browser_task, complete_browser_task, prune_browser_tasks, tasks_dir
from deerflow.config.app_config import AppConfig
from deerflow.config.browser_context_config import BrowserContextConfig
from deerflow.runtime import serialize_channel_values
@@ -307,10 +307,19 @@ async def claim_browser_context_task(
config: AppConfig = Depends(get_config),
) -> BrowserContextTaskNextResponse:
settings = _require_enabled(config)
effective_user_id = get_effective_user_id()
effective_worker_id = worker_id or "chrome-extension"
logger.info(
"Browser task next poll user_id=%s worker_id=%s tasks_dir=%s client=%s",
effective_user_id,
effective_worker_id,
tasks_dir(),
getattr(request.client, "host", None),
)
prune_browser_tasks()
task = claim_next_browser_task(
user_id=get_effective_user_id(),
worker_id=worker_id or "chrome-extension",
user_id=effective_user_id,
worker_id=effective_worker_id,
lease_seconds=settings.task_lease_seconds,
)
if task is None:
@@ -5,7 +5,9 @@
入库目标接口的字段名不固定:标题字段可能叫「名称」也可能叫「标题」,因此先按
「名称」发送一次,若判定失败则用 **同一个 nodeName** 把标题字段换成「标题」再
发一次。nodeName 是入库记录的唯一标识,取「md 标题 + 时间戳」。
发一次。nodeName 是入库记录的唯一标识,**只由 md 标题决定**(md5(标题)),不含
时间戳——同一篇 md(同标题)每次入库得到的 nodeName 都一致,避免重复入库时产生
多条记录。
"""
from __future__ import annotations
@@ -56,7 +58,7 @@ class KnowledgeIngestAttempt(BaseModel):
class KnowledgeIngestPublishResponse(BaseModel):
success: bool
template_name: str
node_name: str = Field(..., description="入库唯一标识(md 标题 + 时间戳),两次尝试共用")
node_name: str = Field(..., description="入库唯一标识(md5(标题),同标题每次入库一致),两次尝试共用")
batch_number: str
attempts: list[KnowledgeIngestAttempt]
@@ -99,10 +101,14 @@ def _build_payload(template_name: str, batch_number: str, node_name: str, title_
}
def _build_node_name(title: str, timestamp: str) -> str:
"""Build a stable node name as md5(title) + timestamp + .md."""
def _build_node_name(title: str) -> str:
"""Build a stable node name = md5(title) + .md.
nodeName 只由标题决定(不含时间戳),所以同一篇 md(同标题)每次入库都得到
完全相同的 nodeName —— 入库接口据此把重复上传当作同一条记录,而不是每次新增。
"""
title_md5 = hashlib.md5(title.encode("utf-8")).hexdigest()
return f"{title_md5}-{timestamp}.md"
return f"{title_md5}.md"
def _resolve_ingest_url() -> str:
@@ -132,10 +138,10 @@ async def publish_to_knowledge_base(request: KnowledgeIngestPublishRequest) -> K
raise HTTPException(status_code=400, detail=f"智能体 '{request.agent_id}' 未配置入库模板名(template_name),无法发布")
now = datetime.now()
# nodeName 是入库记录的唯一标识:md 标题 + 时间戳。两次尝试必须共用同一个
# nodeName,避免第一次实际已入库时第二次产生重复记录。
timestamp = now.strftime("%Y%m%d%H%M%S")
node_name = _build_node_name(title, timestamp)
# nodeName 是入库记录的唯一标识:只取 md5(标题),不含时间戳。同一篇 md(同标题)
# 每次入库得到一致的 nodeName —— 既保证两次尝试(名称/标题字段)共用同一个,也
# 保证重复上传同一篇 md 不会产生多条记录。
node_name = _build_node_name(title)
batch_number = f"md{now.strftime('%Y%m%d')}"
ingest_url = _resolve_ingest_url()
@@ -171,6 +171,11 @@ vision:
# prompt: 请详细描述这张图片的内容。
# max_tokens: 2048
# browser_act 相关配置
browser_context:
enabled: true
actions_enabled: true
# 上传文件识别配置。
# - auto_convert_documents: true 时,上传的 PDF / Word / Excel / PPT 会在服务端自动转换为
# Markdown(.md),agent 通过 read_file / grep 读取其内容 —— 即“识别”文档内容。
@@ -268,6 +273,16 @@ tools:
group: bash
use: deerflow.sandbox.tools:bash_tool
# browser_act 相关配置
- name: browser_fetch_page
group: web
use: deerflow.tools.builtins.browser_context_tool:browser_fetch_page_tool
# browser_act 相关配置
- name: browser_act
group: web
use: deerflow.tools.builtins.browser_act_tool:browser_act_tool
sandbox:
use: deerflow.sandbox.local:LocalSandboxProvider
allow_host_bash: true
@@ -1,14 +1,35 @@
{
"mcpServers": {},
"skills": {
"web-research": { "enabled": true },
"deep-search": { "enabled": true },
"code-retrieval": { "enabled": true },
"news-briefing": { "enabled": true },
"workspace-search": { "enabled": true },
"policy-knowledge": { "enabled": false },
"html-page-builder": { "enabled": true },
"html-charts": { "enabled": true },
"html-multipage": { "enabled": true }
"web-research": {
"enabled": true
},
"deep-search": {
"enabled": true
},
"code-retrieval": {
"enabled": true
},
"news-briefing": {
"enabled": true
},
"workspace-search": {
"enabled": true
},
"policy-knowledge": {
"enabled": false
},
"html-page-builder": {
"enabled": true
},
"html-charts": {
"enabled": true
},
"html-multipage": {
"enabled": true
},
"browser_act": {
"enabled": true
}
}
}
}
@@ -86,27 +86,6 @@ def _drop_archived_skills(skills: list[Skill]) -> list[Skill]:
return skills
def _run_coro_blocking(coro):
"""Run an async coroutine to completion from sync code.
Mirrors ``skill_tools._run_coro_blocking``: when already inside a running
event loop (e.g. prompt built during an async agent run), hand the coroutine
to a fresh loop in a worker thread; otherwise run it directly.
"""
try:
asyncio.get_running_loop()
in_loop = True
except RuntimeError:
in_loop = False
if in_loop:
import concurrent.futures
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
return pool.submit(asyncio.run, coro).result()
return asyncio.run(coro)
def _load_always_on_names() -> set[str]:
"""Names marked 常驻(免压缩), read from the **DB** (skills table).
@@ -114,11 +93,11 @@ def _load_always_on_names() -> set[str]:
simply treats every skill as compressible (safe default).
"""
try:
from deerflow.persistence.engine import get_session_factory
from deerflow.persistence.engine import get_session_factory, run_db_blocking
from deerflow.persistence.skills import make_skill_store
store = make_skill_store(get_session_factory())
return set(_run_coro_blocking(store.list_always_on_names()))
return set(run_db_blocking(store.list_always_on_names()))
except Exception:
logger.debug("Failed to load always_on skill names from DB", exc_info=True)
return set()
@@ -20,7 +20,6 @@ not the retry-wrapped successful response.
from __future__ import annotations
import asyncio
import logging
import time
import uuid
@@ -38,7 +37,7 @@ from langchain.agents.middleware.types import (
from langchain_core.messages import AIMessage
from langgraph.errors import GraphBubbleUp
from deerflow.persistence.engine import get_session_factory
from deerflow.persistence.engine import fire_and_forget_db_write, get_session_factory
from deerflow.persistence.llm_metrics import LlmMetricRecord, make_llm_metrics_store
from deerflow.persistence.llm_metrics.base import LlmMetricsStore
from deerflow.runtime.user_context import get_effective_user_id
@@ -280,16 +279,13 @@ class LlmMetricsMiddleware(AgentMiddleware[AgentState]):
store = self._store()
if store is None:
return
# Route the best-effort write onto the engine's home loop. Doing
# ``asyncio.run(store.record(...))`` here (when the model call ran in a worker
# thread) would reuse the global engine's pooled connections on a fresh loop and
# raise "got Future attached to a different loop" — harmless (caught below) but
# log-spammy and pool-churning under concurrency. See ``fire_and_forget_db_write``.
try:
try:
loop = asyncio.get_running_loop()
except RuntimeError:
loop = None
if loop is not None and loop.is_running():
# Fire-and-forget — never let metric IO block the request path.
loop.create_task(store.record(record))
else:
asyncio.run(store.record(record))
fire_and_forget_db_write(store.record(record))
except Exception:
logger.warning("LlmMetricsMiddleware: failed to flush metric %s", record.id, exc_info=True)
@@ -25,7 +25,6 @@ agent runtime keeps running.
from __future__ import annotations
import asyncio
import logging
import re
import time
@@ -41,7 +40,7 @@ from langgraph.errors import GraphBubbleUp
from langgraph.prebuilt.tool_node import ToolCallRequest
from langgraph.types import Command
from deerflow.persistence.engine import get_session_factory
from deerflow.persistence.engine import fire_and_forget_db_write, get_session_factory
from deerflow.persistence.tool_metrics import ToolMetricRecord, make_tool_metrics_store
from deerflow.persistence.tool_metrics.base import ToolMetricsStore
from deerflow.runtime.user_context import get_effective_user_id
@@ -262,16 +261,12 @@ class ToolMetricsMiddleware(AgentMiddleware[AgentState]):
store = self._store()
if store is None:
return
# Route the best-effort write onto the engine's home loop (a tool call that ran
# in a worker thread would otherwise do ``asyncio.run(store.record(...))`` and reuse
# the global engine's pooled connections cross-loop → "got Future attached to a
# different loop"). See ``fire_and_forget_db_write``.
try:
try:
loop = asyncio.get_running_loop()
except RuntimeError:
loop = None
if loop is not None and loop.is_running():
# Fire-and-forget — never let metric IO block the tool path.
loop.create_task(store.record(record))
else:
asyncio.run(store.record(record))
fire_and_forget_db_write(store.record(record))
except Exception:
logger.warning("ToolMetricsMiddleware: failed to flush metric %s", record.id, exc_info=True)
@@ -193,16 +193,22 @@ def claim_next_browser_task(
user_id = str(user_id or "default")
worker_id = (worker_id or "chrome-extension").strip()[:128] or "chrome-extension"
lease_seconds = max(10, int(lease_seconds))
queue_dir = tasks_dir()
with _LOCK:
paths = sorted(tasks_dir().glob("*.json"), key=lambda p: p.stat().st_mtime)
paths = sorted(queue_dir.glob("*.json"), key=lambda p: p.stat().st_mtime)
matching_user = 0
matching_status_counts: dict[str, int] = {}
for path in paths:
data = _read_task(path)
if not data or str(data.get("user_id") or "") != user_id:
continue
matching_user += 1
data = _expire_if_needed(path, data, now)
status = data.get("status")
status_key = str(status or "unknown")
matching_status_counts[status_key] = matching_status_counts.get(status_key, 0) + 1
if status == "running":
updated_at = _parse_iso(data.get("updated_at")) or now
if (now - updated_at).total_seconds() <= lease_seconds:
@@ -220,8 +226,32 @@ def claim_next_browser_task(
data["claimed_at"] = _iso(now)
data["updated_at"] = _iso(now)
_write_task(path, data)
logger.info(
"Browser task claim scan user_id=%s worker_id=%s tasks_dir=%s total_files=%d matching_user=%d "
"matching_statuses=%s claimed_task_id=%s claimed_kind=%s",
user_id,
worker_id,
queue_dir,
len(paths),
matching_user,
matching_status_counts,
data.get("id"),
data.get("kind") or "fetch",
)
return _public_task(data)
logger.info(
"Browser task claim scan user_id=%s worker_id=%s tasks_dir=%s total_files=%d matching_user=%d "
"matching_statuses=%s claimed_task_id=%s claimed_kind=%s",
user_id,
worker_id,
queue_dir,
len(paths),
matching_user,
matching_status_counts,
None,
None,
)
return None
@@ -300,4 +330,3 @@ async def wait_for_browser_task_result(
if time.monotonic() >= deadline:
return task
await asyncio.sleep(interval)
@@ -32,6 +32,13 @@ logger = logging.getLogger(__name__)
_engine: AsyncEngine | None = None
_session_factory: async_sessionmaker[AsyncSession] | None = None
# The event loop on which the async engine was created. asyncmy/asyncpg connection
# pools bind their connections to the loop that opened them; awaiting a write on a
# *different* loop reuses those connections cross-loop and raises
# "got Future attached to a different loop". We capture the engine's home loop here so
# best-effort writes coming from worker threads / other loops can be routed back to it
# (see ``fire_and_forget_db_write``).
_engine_loop: asyncio.AbstractEventLoop | None = None
def _add_runtime_vendor_path() -> None:
@@ -293,12 +300,20 @@ async def init_engine(
pool_timeout: Seconds to wait for a free pooled connection before erroring (postgres/mysql).
sqlite_dir: Directory to create for SQLite (ensured to exist).
"""
global _engine, _session_factory
global _engine, _session_factory, _engine_loop
if backend == "memory":
logger.info("Persistence backend=memory -- ORM engine not initialized")
return
# Remember the loop that owns the engine's connection pool, so off-loop
# best-effort writes (metrics from worker threads, etc.) can be routed here
# instead of reusing pooled connections cross-loop.
try:
_engine_loop = asyncio.get_running_loop()
except RuntimeError:
_engine_loop = None
if backend == "postgres":
try:
import asyncpg # noqa: F401
@@ -459,11 +474,121 @@ def get_engine() -> AsyncEngine | None:
return _engine
def get_engine_loop() -> asyncio.AbstractEventLoop | None:
"""Return the event loop that owns the engine's connection pool (or None)."""
return _engine_loop
def fire_and_forget_db_write(coro) -> None:
"""Schedule a best-effort DB write on the engine's home loop, from any loop/thread.
The async engine's connection pool is bound to the loop that created it
(``init_engine``, normally the Gateway's main loop). Running a write on a
*different* loop — e.g. a model/tool call that LangGraph executed in a worker
thread, which then does ``asyncio.run(store.record(...))`` — reuses those pooled
connections cross-loop and raises ``got Future attached to a different loop``.
Under concurrency this happens constantly (many calls land in worker threads at
once), spamming the log and churning the pool; single-threaded it almost never does.
This routes every such write back onto the engine's home loop:
- already on that loop -> ``create_task`` (in-loop fire-and-forget);
- on another loop/thread -> ``run_coroutine_threadsafe`` to the home loop.
The write itself stays best-effort: if no engine loop is captured (memory backend
/ not initialized) the coroutine is closed (avoids a "never awaited" warning) and
the call is a no-op. Never raises.
"""
loop = _engine_loop
if loop is None or loop.is_closed():
close = getattr(coro, "close", None)
if callable(close):
close()
return
try:
running = asyncio.get_running_loop()
except RuntimeError:
running = None
try:
if running is loop:
loop.create_task(coro)
else:
asyncio.run_coroutine_threadsafe(coro, loop)
except Exception: # noqa: BLE001 — best-effort; scheduling must never break the caller
close = getattr(coro, "close", None)
if callable(close):
close()
logger.debug("fire_and_forget_db_write: failed to schedule DB write", exc_info=True)
def _run_coro_on_throwaway_loop(coro):
"""Last-resort: run *coro* to completion on a fresh event loop.
Only reached when there is no usable engine home loop, or when the caller is
*synchronously* inside the home loop (routing back would deadlock). This
touches the shared connection pool from a throwaway loop, so it can still leak
a dead-loop-bound connection — but the common worker-thread path no longer
gets here (see ``run_db_blocking``).
"""
try:
asyncio.get_running_loop()
in_loop = True
except RuntimeError:
in_loop = False
if in_loop:
import concurrent.futures
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
return pool.submit(asyncio.run, coro).result()
return asyncio.run(coro)
def run_db_blocking(coro, *, timeout: float | None = 30.0):
"""Run an awaited DB coroutine from sync code, safe w.r.t. the engine's home loop.
The blocking-read sibling of :func:`fire_and_forget_db_write`: where that one
drops the result, this returns it — used by the sync→async bridges that must
read a row before proceeding (thread-path owner resolution, always-on skills,
skill visibility).
The async engine's connection pool is bound to the loop that created it
(``init_engine``). Running a DB coroutine on a *throwaway* loop
(``asyncio.run``) checks out pooled connections cross-loop and **leaks** them:
the connection ends up bound to a loop that is then closed, and when the pool
later recycles/disposes it, asyncmy's ``_terminate_graceful_close`` fires on the
wrong loop and raises ``got Future <...> attached to a different loop``. Under
concurrency (scheduler / subagent / sandbox-tool worker threads each spinning a
fresh loop) this floods the logs.
So when we are **off** the home loop — the common case, a worker thread with no
running loop — route the coroutine onto the home loop and block the *caller*
thread for its result. The pool is only ever touched on its own loop, so nothing
leaks. Only when there is no home loop, or we are synchronously inside it (where
blocking would deadlock), do we fall back to a throwaway loop.
Never swallows the coroutine's own exception — the sync bridges wrap this call in
their own try/except and degrade gracefully.
"""
loop = _engine_loop
try:
running = asyncio.get_running_loop()
except RuntimeError:
running = None
if loop is not None and not loop.is_closed() and running is not loop:
# Off the home loop (worker thread, or a different loop): execute the coro
# on the engine's loop and block this thread for the result. asyncmy's
# pooled connections stay on their own loop, so none leak cross-loop.
return asyncio.run_coroutine_threadsafe(coro, loop).result(timeout)
# No home loop, or we're already synchronously inside it: last resort.
return _run_coro_on_throwaway_loop(coro)
async def close_engine() -> None:
"""Dispose the engine, release all connections."""
global _engine, _session_factory
global _engine, _session_factory, _engine_loop
if _engine is not None:
await _engine.dispose()
logger.info("Persistence engine closed")
_engine = None
_session_factory = None
_engine_loop = None
@@ -28,7 +28,6 @@ safely to the pre-fix semantics.
from __future__ import annotations
import asyncio
import logging
import threading
@@ -106,27 +105,6 @@ def _is_creator_shared_thread(metadata: dict) -> bool:
return bool(metadata.get("taskId")) or metadata.get("thread_type") == "roundtable"
def _run_coro_blocking(coro):
"""Run an async coroutine to completion from sync code.
Mirrors ``lead_agent.prompt._run_coro_blocking``: when already inside a
running event loop, hand the coroutine to a fresh loop in a worker thread;
otherwise run it directly.
"""
try:
asyncio.get_running_loop()
in_loop = True
except RuntimeError:
in_loop = False
if in_loop:
import concurrent.futures
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
return pool.submit(asyncio.run, coro).result()
return asyncio.run(coro)
def resolve_path_user_id(thread_id: str | None) -> str:
"""Canonical user bucket for *thread_id*'s files (sync).
@@ -140,7 +118,9 @@ def resolve_path_user_id(thread_id: str | None) -> str:
if cached is not None:
return cached
try:
owner = _run_coro_blocking(_fetch_task_owner(thread_id))
from deerflow.persistence.engine import run_db_blocking
owner = run_db_blocking(_fetch_task_owner(thread_id))
except Exception:
logger.debug("Failed to resolve task-shared owner for thread %s", thread_id, exc_info=True)
owner = None
@@ -75,30 +75,6 @@ def _resolve_effective_user_id() -> str | None:
return None
def _run_coro_blocking(coro):
"""在同步工具里把一个协程跑到完成。
``skill_list`` 是同步 ``@tool``:LangChain 通常在线程池里执行它(无运行中的
事件循环),但 curator 评审会在已有事件循环的线程里调用它。两种情况都要能
拿到 ``SkillStore`` 的异步查询结果 —— 检测到运行中的事件循环就转交独立线程
执行 ``asyncio.run``,否则直接跑。
"""
import asyncio
try:
asyncio.get_running_loop()
in_loop = True
except RuntimeError:
in_loop = False
if in_loop:
import concurrent.futures
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
return pool.submit(asyncio.run, coro).result()
return asyncio.run(coro)
def _load_owned_skill_names(user_id: str | None) -> set[str] | None:
"""返回 ``owner_user_id == user_id`` 的自定义技能名集合。
@@ -110,14 +86,14 @@ def _load_owned_skill_names(user_id: str | None) -> set[str] | None:
if not user_id:
return None
try:
from deerflow.persistence.engine import get_session_factory
from deerflow.persistence.engine import get_session_factory, run_db_blocking
from deerflow.persistence.skills import make_skill_store
factory = get_session_factory()
if factory is None:
return None
store = make_skill_store(factory)
records = _run_coro_blocking(store.list_visible(user_id))
records = run_db_blocking(store.list_visible(user_id))
return {r["name"] for r in records if r.get("owner_user_id") == user_id}
except Exception:
logger.debug("Failed to load owned skill names for skill_list filter", exc_info=True)
@@ -3,6 +3,7 @@
from __future__ import annotations
import asyncio
import logging
import pytest
@@ -76,6 +77,22 @@ def test_fetch_task_default_kind_is_fetch():
assert claimed["action"] is None
def test_claim_next_browser_task_logs_claim_diagnostics(caplog):
task_queue.create_browser_task(user_id="u1", url="https://example.com/1")
task_queue.create_browser_task(user_id="u2", url="https://example.com/2")
with caplog.at_level(logging.INFO, logger=task_queue.__name__):
claimed = task_queue.claim_next_browser_task(user_id="u1", worker_id="worker-1")
assert claimed is not None
assert "Browser task claim scan" in caplog.text
assert "user_id=u1" in caplog.text
assert "worker_id=worker-1" in caplog.text
assert "total_files=2" in caplog.text
assert "matching_user=1" in caplog.text
assert "claimed_task_id=" in caplog.text
# --- tool validation --------------------------------------------------------