Merge branch 'master' of http://192.168.0.200/knowledge-base-bj/deerflow-server into dhk-1
This commit is contained in:
commit
3ea8e73389
61 files changed
+734
-429
No files matched your search
@@ -27,7 +27,7 @@ from app.gateway.routers.thread_runs import RunCreateRequest
|
||||
from app.gateway.routers.threads import ThreadCreateRequest, create_thread
|
||||
from app.gateway.routers.uploads import _make_file_sandbox_writable
|
||||
from app.gateway.services import start_run
|
||||
from deerflow.browser_context.tasks import claim_next_browser_task, complete_browser_task, prune_browser_tasks
|
||||
from deerflow.browser_context.tasks import claim_next_browser_task, complete_browser_task, prune_browser_tasks, tasks_dir
|
||||
from deerflow.config.app_config import AppConfig
|
||||
from deerflow.config.browser_context_config import BrowserContextConfig
|
||||
from deerflow.runtime import serialize_channel_values
|
||||
@@ -307,10 +307,19 @@ async def claim_browser_context_task(
|
||||
config: AppConfig = Depends(get_config),
|
||||
) -> BrowserContextTaskNextResponse:
|
||||
settings = _require_enabled(config)
|
||||
effective_user_id = get_effective_user_id()
|
||||
effective_worker_id = worker_id or "chrome-extension"
|
||||
logger.info(
|
||||
"Browser task next poll user_id=%s worker_id=%s tasks_dir=%s client=%s",
|
||||
effective_user_id,
|
||||
effective_worker_id,
|
||||
tasks_dir(),
|
||||
getattr(request.client, "host", None),
|
||||
)
|
||||
prune_browser_tasks()
|
||||
task = claim_next_browser_task(
|
||||
user_id=get_effective_user_id(),
|
||||
worker_id=worker_id or "chrome-extension",
|
||||
user_id=effective_user_id,
|
||||
worker_id=effective_worker_id,
|
||||
lease_seconds=settings.task_lease_seconds,
|
||||
)
|
||||
if task is None:
|
||||
|
||||
@@ -5,7 +5,9 @@
|
||||
|
||||
入库目标接口的字段名不固定:标题字段可能叫「名称」也可能叫「标题」,因此先按
|
||||
「名称」发送一次,若判定失败则用 **同一个 nodeName** 把标题字段换成「标题」再
|
||||
发一次。nodeName 是入库记录的唯一标识,取「md 标题 + 时间戳」。
|
||||
发一次。nodeName 是入库记录的唯一标识,**只由 md 标题决定**(md5(标题)),不含
|
||||
时间戳——同一篇 md(同标题)每次入库得到的 nodeName 都一致,避免重复入库时产生
|
||||
多条记录。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -56,7 +58,7 @@ class KnowledgeIngestAttempt(BaseModel):
|
||||
class KnowledgeIngestPublishResponse(BaseModel):
|
||||
success: bool
|
||||
template_name: str
|
||||
node_name: str = Field(..., description="入库唯一标识(md 标题 + 时间戳),两次尝试共用")
|
||||
node_name: str = Field(..., description="入库唯一标识(md5(标题),同标题每次入库一致),两次尝试共用")
|
||||
batch_number: str
|
||||
attempts: list[KnowledgeIngestAttempt]
|
||||
|
||||
@@ -99,10 +101,14 @@ def _build_payload(template_name: str, batch_number: str, node_name: str, title_
|
||||
}
|
||||
|
||||
|
||||
def _build_node_name(title: str, timestamp: str) -> str:
|
||||
"""Build a stable node name as md5(title) + timestamp + .md."""
|
||||
def _build_node_name(title: str) -> str:
|
||||
"""Build a stable node name = md5(title) + .md.
|
||||
|
||||
nodeName 只由标题决定(不含时间戳),所以同一篇 md(同标题)每次入库都得到
|
||||
完全相同的 nodeName —— 入库接口据此把重复上传当作同一条记录,而不是每次新增。
|
||||
"""
|
||||
title_md5 = hashlib.md5(title.encode("utf-8")).hexdigest()
|
||||
return f"{title_md5}-{timestamp}.md"
|
||||
return f"{title_md5}.md"
|
||||
|
||||
|
||||
def _resolve_ingest_url() -> str:
|
||||
@@ -132,10 +138,10 @@ async def publish_to_knowledge_base(request: KnowledgeIngestPublishRequest) -> K
|
||||
raise HTTPException(status_code=400, detail=f"智能体 '{request.agent_id}' 未配置入库模板名(template_name),无法发布")
|
||||
|
||||
now = datetime.now()
|
||||
# nodeName 是入库记录的唯一标识:md 标题 + 时间戳。两次尝试必须共用同一个
|
||||
# nodeName,避免第一次实际已入库时第二次产生重复记录。
|
||||
timestamp = now.strftime("%Y%m%d%H%M%S")
|
||||
node_name = _build_node_name(title, timestamp)
|
||||
# nodeName 是入库记录的唯一标识:只取 md5(标题),不含时间戳。同一篇 md(同标题)
|
||||
# 每次入库得到一致的 nodeName —— 既保证两次尝试(名称/标题字段)共用同一个,也
|
||||
# 保证重复上传同一篇 md 不会产生多条记录。
|
||||
node_name = _build_node_name(title)
|
||||
batch_number = f"md{now.strftime('%Y%m%d')}"
|
||||
ingest_url = _resolve_ingest_url()
|
||||
|
||||
|
||||
@@ -171,6 +171,11 @@ vision:
|
||||
# prompt: 请详细描述这张图片的内容。
|
||||
# max_tokens: 2048
|
||||
|
||||
# browser_act 相关配置
|
||||
browser_context:
|
||||
enabled: true
|
||||
actions_enabled: true
|
||||
|
||||
# 上传文件识别配置。
|
||||
# - auto_convert_documents: true 时,上传的 PDF / Word / Excel / PPT 会在服务端自动转换为
|
||||
# Markdown(.md),agent 通过 read_file / grep 读取其内容 —— 即“识别”文档内容。
|
||||
@@ -268,6 +273,16 @@ tools:
|
||||
group: bash
|
||||
use: deerflow.sandbox.tools:bash_tool
|
||||
|
||||
# browser_act 相关配置
|
||||
- name: browser_fetch_page
|
||||
group: web
|
||||
use: deerflow.tools.builtins.browser_context_tool:browser_fetch_page_tool
|
||||
|
||||
# browser_act 相关配置
|
||||
- name: browser_act
|
||||
group: web
|
||||
use: deerflow.tools.builtins.browser_act_tool:browser_act_tool
|
||||
|
||||
sandbox:
|
||||
use: deerflow.sandbox.local:LocalSandboxProvider
|
||||
allow_host_bash: true
|
||||
|
||||
@@ -1,14 +1,35 @@
|
||||
{
|
||||
"mcpServers": {},
|
||||
"skills": {
|
||||
"web-research": { "enabled": true },
|
||||
"deep-search": { "enabled": true },
|
||||
"code-retrieval": { "enabled": true },
|
||||
"news-briefing": { "enabled": true },
|
||||
"workspace-search": { "enabled": true },
|
||||
"policy-knowledge": { "enabled": false },
|
||||
"html-page-builder": { "enabled": true },
|
||||
"html-charts": { "enabled": true },
|
||||
"html-multipage": { "enabled": true }
|
||||
"web-research": {
|
||||
"enabled": true
|
||||
},
|
||||
"deep-search": {
|
||||
"enabled": true
|
||||
},
|
||||
"code-retrieval": {
|
||||
"enabled": true
|
||||
},
|
||||
"news-briefing": {
|
||||
"enabled": true
|
||||
},
|
||||
"workspace-search": {
|
||||
"enabled": true
|
||||
},
|
||||
"policy-knowledge": {
|
||||
"enabled": false
|
||||
},
|
||||
"html-page-builder": {
|
||||
"enabled": true
|
||||
},
|
||||
"html-charts": {
|
||||
"enabled": true
|
||||
},
|
||||
"html-multipage": {
|
||||
"enabled": true
|
||||
},
|
||||
"browser_act": {
|
||||
"enabled": true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+2
-23
@@ -86,27 +86,6 @@ def _drop_archived_skills(skills: list[Skill]) -> list[Skill]:
|
||||
return skills
|
||||
|
||||
|
||||
def _run_coro_blocking(coro):
|
||||
"""Run an async coroutine to completion from sync code.
|
||||
|
||||
Mirrors ``skill_tools._run_coro_blocking``: when already inside a running
|
||||
event loop (e.g. prompt built during an async agent run), hand the coroutine
|
||||
to a fresh loop in a worker thread; otherwise run it directly.
|
||||
"""
|
||||
try:
|
||||
asyncio.get_running_loop()
|
||||
in_loop = True
|
||||
except RuntimeError:
|
||||
in_loop = False
|
||||
|
||||
if in_loop:
|
||||
import concurrent.futures
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
|
||||
return pool.submit(asyncio.run, coro).result()
|
||||
return asyncio.run(coro)
|
||||
|
||||
|
||||
def _load_always_on_names() -> set[str]:
|
||||
"""Names marked 常驻(免压缩), read from the **DB** (skills table).
|
||||
|
||||
@@ -114,11 +93,11 @@ def _load_always_on_names() -> set[str]:
|
||||
simply treats every skill as compressible (safe default).
|
||||
"""
|
||||
try:
|
||||
from deerflow.persistence.engine import get_session_factory
|
||||
from deerflow.persistence.engine import get_session_factory, run_db_blocking
|
||||
from deerflow.persistence.skills import make_skill_store
|
||||
|
||||
store = make_skill_store(get_session_factory())
|
||||
return set(_run_coro_blocking(store.list_always_on_names()))
|
||||
return set(run_db_blocking(store.list_always_on_names()))
|
||||
except Exception:
|
||||
logger.debug("Failed to load always_on skill names from DB", exc_info=True)
|
||||
return set()
|
||||
|
||||
+7
-11
@@ -20,7 +20,6 @@ not the retry-wrapped successful response.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
import uuid
|
||||
@@ -38,7 +37,7 @@ from langchain.agents.middleware.types import (
|
||||
from langchain_core.messages import AIMessage
|
||||
from langgraph.errors import GraphBubbleUp
|
||||
|
||||
from deerflow.persistence.engine import get_session_factory
|
||||
from deerflow.persistence.engine import fire_and_forget_db_write, get_session_factory
|
||||
from deerflow.persistence.llm_metrics import LlmMetricRecord, make_llm_metrics_store
|
||||
from deerflow.persistence.llm_metrics.base import LlmMetricsStore
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
@@ -280,16 +279,13 @@ class LlmMetricsMiddleware(AgentMiddleware[AgentState]):
|
||||
store = self._store()
|
||||
if store is None:
|
||||
return
|
||||
# Route the best-effort write onto the engine's home loop. Doing
|
||||
# ``asyncio.run(store.record(...))`` here (when the model call ran in a worker
|
||||
# thread) would reuse the global engine's pooled connections on a fresh loop and
|
||||
# raise "got Future attached to a different loop" — harmless (caught below) but
|
||||
# log-spammy and pool-churning under concurrency. See ``fire_and_forget_db_write``.
|
||||
try:
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
loop = None
|
||||
if loop is not None and loop.is_running():
|
||||
# Fire-and-forget — never let metric IO block the request path.
|
||||
loop.create_task(store.record(record))
|
||||
else:
|
||||
asyncio.run(store.record(record))
|
||||
fire_and_forget_db_write(store.record(record))
|
||||
except Exception:
|
||||
logger.warning("LlmMetricsMiddleware: failed to flush metric %s", record.id, exc_info=True)
|
||||
|
||||
|
||||
+6
-11
@@ -25,7 +25,6 @@ agent runtime keeps running.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
@@ -41,7 +40,7 @@ from langgraph.errors import GraphBubbleUp
|
||||
from langgraph.prebuilt.tool_node import ToolCallRequest
|
||||
from langgraph.types import Command
|
||||
|
||||
from deerflow.persistence.engine import get_session_factory
|
||||
from deerflow.persistence.engine import fire_and_forget_db_write, get_session_factory
|
||||
from deerflow.persistence.tool_metrics import ToolMetricRecord, make_tool_metrics_store
|
||||
from deerflow.persistence.tool_metrics.base import ToolMetricsStore
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
@@ -262,16 +261,12 @@ class ToolMetricsMiddleware(AgentMiddleware[AgentState]):
|
||||
store = self._store()
|
||||
if store is None:
|
||||
return
|
||||
# Route the best-effort write onto the engine's home loop (a tool call that ran
|
||||
# in a worker thread would otherwise do ``asyncio.run(store.record(...))`` and reuse
|
||||
# the global engine's pooled connections cross-loop → "got Future attached to a
|
||||
# different loop"). See ``fire_and_forget_db_write``.
|
||||
try:
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
loop = None
|
||||
if loop is not None and loop.is_running():
|
||||
# Fire-and-forget — never let metric IO block the tool path.
|
||||
loop.create_task(store.record(record))
|
||||
else:
|
||||
asyncio.run(store.record(record))
|
||||
fire_and_forget_db_write(store.record(record))
|
||||
except Exception:
|
||||
logger.warning("ToolMetricsMiddleware: failed to flush metric %s", record.id, exc_info=True)
|
||||
|
||||
|
||||
+31
-2
@@ -193,16 +193,22 @@ def claim_next_browser_task(
|
||||
user_id = str(user_id or "default")
|
||||
worker_id = (worker_id or "chrome-extension").strip()[:128] or "chrome-extension"
|
||||
lease_seconds = max(10, int(lease_seconds))
|
||||
queue_dir = tasks_dir()
|
||||
|
||||
with _LOCK:
|
||||
paths = sorted(tasks_dir().glob("*.json"), key=lambda p: p.stat().st_mtime)
|
||||
paths = sorted(queue_dir.glob("*.json"), key=lambda p: p.stat().st_mtime)
|
||||
matching_user = 0
|
||||
matching_status_counts: dict[str, int] = {}
|
||||
for path in paths:
|
||||
data = _read_task(path)
|
||||
if not data or str(data.get("user_id") or "") != user_id:
|
||||
continue
|
||||
matching_user += 1
|
||||
|
||||
data = _expire_if_needed(path, data, now)
|
||||
status = data.get("status")
|
||||
status_key = str(status or "unknown")
|
||||
matching_status_counts[status_key] = matching_status_counts.get(status_key, 0) + 1
|
||||
if status == "running":
|
||||
updated_at = _parse_iso(data.get("updated_at")) or now
|
||||
if (now - updated_at).total_seconds() <= lease_seconds:
|
||||
@@ -220,8 +226,32 @@ def claim_next_browser_task(
|
||||
data["claimed_at"] = _iso(now)
|
||||
data["updated_at"] = _iso(now)
|
||||
_write_task(path, data)
|
||||
logger.info(
|
||||
"Browser task claim scan user_id=%s worker_id=%s tasks_dir=%s total_files=%d matching_user=%d "
|
||||
"matching_statuses=%s claimed_task_id=%s claimed_kind=%s",
|
||||
user_id,
|
||||
worker_id,
|
||||
queue_dir,
|
||||
len(paths),
|
||||
matching_user,
|
||||
matching_status_counts,
|
||||
data.get("id"),
|
||||
data.get("kind") or "fetch",
|
||||
)
|
||||
return _public_task(data)
|
||||
|
||||
logger.info(
|
||||
"Browser task claim scan user_id=%s worker_id=%s tasks_dir=%s total_files=%d matching_user=%d "
|
||||
"matching_statuses=%s claimed_task_id=%s claimed_kind=%s",
|
||||
user_id,
|
||||
worker_id,
|
||||
queue_dir,
|
||||
len(paths),
|
||||
matching_user,
|
||||
matching_status_counts,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
@@ -300,4 +330,3 @@ async def wait_for_browser_task_result(
|
||||
if time.monotonic() >= deadline:
|
||||
return task
|
||||
await asyncio.sleep(interval)
|
||||
|
||||
@@ -32,6 +32,13 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
_engine: AsyncEngine | None = None
|
||||
_session_factory: async_sessionmaker[AsyncSession] | None = None
|
||||
# The event loop on which the async engine was created. asyncmy/asyncpg connection
|
||||
# pools bind their connections to the loop that opened them; awaiting a write on a
|
||||
# *different* loop reuses those connections cross-loop and raises
|
||||
# "got Future attached to a different loop". We capture the engine's home loop here so
|
||||
# best-effort writes coming from worker threads / other loops can be routed back to it
|
||||
# (see ``fire_and_forget_db_write``).
|
||||
_engine_loop: asyncio.AbstractEventLoop | None = None
|
||||
|
||||
|
||||
def _add_runtime_vendor_path() -> None:
|
||||
@@ -293,12 +300,20 @@ async def init_engine(
|
||||
pool_timeout: Seconds to wait for a free pooled connection before erroring (postgres/mysql).
|
||||
sqlite_dir: Directory to create for SQLite (ensured to exist).
|
||||
"""
|
||||
global _engine, _session_factory
|
||||
global _engine, _session_factory, _engine_loop
|
||||
|
||||
if backend == "memory":
|
||||
logger.info("Persistence backend=memory -- ORM engine not initialized")
|
||||
return
|
||||
|
||||
# Remember the loop that owns the engine's connection pool, so off-loop
|
||||
# best-effort writes (metrics from worker threads, etc.) can be routed here
|
||||
# instead of reusing pooled connections cross-loop.
|
||||
try:
|
||||
_engine_loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
_engine_loop = None
|
||||
|
||||
if backend == "postgres":
|
||||
try:
|
||||
import asyncpg # noqa: F401
|
||||
@@ -459,11 +474,121 @@ def get_engine() -> AsyncEngine | None:
|
||||
return _engine
|
||||
|
||||
|
||||
def get_engine_loop() -> asyncio.AbstractEventLoop | None:
|
||||
"""Return the event loop that owns the engine's connection pool (or None)."""
|
||||
return _engine_loop
|
||||
|
||||
|
||||
def fire_and_forget_db_write(coro) -> None:
|
||||
"""Schedule a best-effort DB write on the engine's home loop, from any loop/thread.
|
||||
|
||||
The async engine's connection pool is bound to the loop that created it
|
||||
(``init_engine``, normally the Gateway's main loop). Running a write on a
|
||||
*different* loop — e.g. a model/tool call that LangGraph executed in a worker
|
||||
thread, which then does ``asyncio.run(store.record(...))`` — reuses those pooled
|
||||
connections cross-loop and raises ``got Future attached to a different loop``.
|
||||
Under concurrency this happens constantly (many calls land in worker threads at
|
||||
once), spamming the log and churning the pool; single-threaded it almost never does.
|
||||
|
||||
This routes every such write back onto the engine's home loop:
|
||||
- already on that loop -> ``create_task`` (in-loop fire-and-forget);
|
||||
- on another loop/thread -> ``run_coroutine_threadsafe`` to the home loop.
|
||||
|
||||
The write itself stays best-effort: if no engine loop is captured (memory backend
|
||||
/ not initialized) the coroutine is closed (avoids a "never awaited" warning) and
|
||||
the call is a no-op. Never raises.
|
||||
"""
|
||||
loop = _engine_loop
|
||||
if loop is None or loop.is_closed():
|
||||
close = getattr(coro, "close", None)
|
||||
if callable(close):
|
||||
close()
|
||||
return
|
||||
try:
|
||||
running = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
running = None
|
||||
try:
|
||||
if running is loop:
|
||||
loop.create_task(coro)
|
||||
else:
|
||||
asyncio.run_coroutine_threadsafe(coro, loop)
|
||||
except Exception: # noqa: BLE001 — best-effort; scheduling must never break the caller
|
||||
close = getattr(coro, "close", None)
|
||||
if callable(close):
|
||||
close()
|
||||
logger.debug("fire_and_forget_db_write: failed to schedule DB write", exc_info=True)
|
||||
|
||||
|
||||
def _run_coro_on_throwaway_loop(coro):
|
||||
"""Last-resort: run *coro* to completion on a fresh event loop.
|
||||
|
||||
Only reached when there is no usable engine home loop, or when the caller is
|
||||
*synchronously* inside the home loop (routing back would deadlock). This
|
||||
touches the shared connection pool from a throwaway loop, so it can still leak
|
||||
a dead-loop-bound connection — but the common worker-thread path no longer
|
||||
gets here (see ``run_db_blocking``).
|
||||
"""
|
||||
try:
|
||||
asyncio.get_running_loop()
|
||||
in_loop = True
|
||||
except RuntimeError:
|
||||
in_loop = False
|
||||
|
||||
if in_loop:
|
||||
import concurrent.futures
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
|
||||
return pool.submit(asyncio.run, coro).result()
|
||||
return asyncio.run(coro)
|
||||
|
||||
|
||||
def run_db_blocking(coro, *, timeout: float | None = 30.0):
|
||||
"""Run an awaited DB coroutine from sync code, safe w.r.t. the engine's home loop.
|
||||
|
||||
The blocking-read sibling of :func:`fire_and_forget_db_write`: where that one
|
||||
drops the result, this returns it — used by the sync→async bridges that must
|
||||
read a row before proceeding (thread-path owner resolution, always-on skills,
|
||||
skill visibility).
|
||||
|
||||
The async engine's connection pool is bound to the loop that created it
|
||||
(``init_engine``). Running a DB coroutine on a *throwaway* loop
|
||||
(``asyncio.run``) checks out pooled connections cross-loop and **leaks** them:
|
||||
the connection ends up bound to a loop that is then closed, and when the pool
|
||||
later recycles/disposes it, asyncmy's ``_terminate_graceful_close`` fires on the
|
||||
wrong loop and raises ``got Future <...> attached to a different loop``. Under
|
||||
concurrency (scheduler / subagent / sandbox-tool worker threads each spinning a
|
||||
fresh loop) this floods the logs.
|
||||
|
||||
So when we are **off** the home loop — the common case, a worker thread with no
|
||||
running loop — route the coroutine onto the home loop and block the *caller*
|
||||
thread for its result. The pool is only ever touched on its own loop, so nothing
|
||||
leaks. Only when there is no home loop, or we are synchronously inside it (where
|
||||
blocking would deadlock), do we fall back to a throwaway loop.
|
||||
|
||||
Never swallows the coroutine's own exception — the sync bridges wrap this call in
|
||||
their own try/except and degrade gracefully.
|
||||
"""
|
||||
loop = _engine_loop
|
||||
try:
|
||||
running = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
running = None
|
||||
if loop is not None and not loop.is_closed() and running is not loop:
|
||||
# Off the home loop (worker thread, or a different loop): execute the coro
|
||||
# on the engine's loop and block this thread for the result. asyncmy's
|
||||
# pooled connections stay on their own loop, so none leak cross-loop.
|
||||
return asyncio.run_coroutine_threadsafe(coro, loop).result(timeout)
|
||||
# No home loop, or we're already synchronously inside it: last resort.
|
||||
return _run_coro_on_throwaway_loop(coro)
|
||||
|
||||
|
||||
async def close_engine() -> None:
|
||||
"""Dispose the engine, release all connections."""
|
||||
global _engine, _session_factory
|
||||
global _engine, _session_factory, _engine_loop
|
||||
if _engine is not None:
|
||||
await _engine.dispose()
|
||||
logger.info("Persistence engine closed")
|
||||
_engine = None
|
||||
_session_factory = None
|
||||
_engine_loop = None
|
||||
@@ -28,7 +28,6 @@ safely to the pre-fix semantics.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import threading
|
||||
|
||||
@@ -106,27 +105,6 @@ def _is_creator_shared_thread(metadata: dict) -> bool:
|
||||
return bool(metadata.get("taskId")) or metadata.get("thread_type") == "roundtable"
|
||||
|
||||
|
||||
def _run_coro_blocking(coro):
|
||||
"""Run an async coroutine to completion from sync code.
|
||||
|
||||
Mirrors ``lead_agent.prompt._run_coro_blocking``: when already inside a
|
||||
running event loop, hand the coroutine to a fresh loop in a worker thread;
|
||||
otherwise run it directly.
|
||||
"""
|
||||
try:
|
||||
asyncio.get_running_loop()
|
||||
in_loop = True
|
||||
except RuntimeError:
|
||||
in_loop = False
|
||||
|
||||
if in_loop:
|
||||
import concurrent.futures
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
|
||||
return pool.submit(asyncio.run, coro).result()
|
||||
return asyncio.run(coro)
|
||||
|
||||
|
||||
def resolve_path_user_id(thread_id: str | None) -> str:
|
||||
"""Canonical user bucket for *thread_id*'s files (sync).
|
||||
|
||||
@@ -140,7 +118,9 @@ def resolve_path_user_id(thread_id: str | None) -> str:
|
||||
if cached is not None:
|
||||
return cached
|
||||
try:
|
||||
owner = _run_coro_blocking(_fetch_task_owner(thread_id))
|
||||
from deerflow.persistence.engine import run_db_blocking
|
||||
|
||||
owner = run_db_blocking(_fetch_task_owner(thread_id))
|
||||
except Exception:
|
||||
logger.debug("Failed to resolve task-shared owner for thread %s", thread_id, exc_info=True)
|
||||
owner = None
|
||||
|
||||
+2
-26
@@ -75,30 +75,6 @@ def _resolve_effective_user_id() -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def _run_coro_blocking(coro):
|
||||
"""在同步工具里把一个协程跑到完成。
|
||||
|
||||
``skill_list`` 是同步 ``@tool``:LangChain 通常在线程池里执行它(无运行中的
|
||||
事件循环),但 curator 评审会在已有事件循环的线程里调用它。两种情况都要能
|
||||
拿到 ``SkillStore`` 的异步查询结果 —— 检测到运行中的事件循环就转交独立线程
|
||||
执行 ``asyncio.run``,否则直接跑。
|
||||
"""
|
||||
import asyncio
|
||||
|
||||
try:
|
||||
asyncio.get_running_loop()
|
||||
in_loop = True
|
||||
except RuntimeError:
|
||||
in_loop = False
|
||||
|
||||
if in_loop:
|
||||
import concurrent.futures
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
|
||||
return pool.submit(asyncio.run, coro).result()
|
||||
return asyncio.run(coro)
|
||||
|
||||
|
||||
def _load_owned_skill_names(user_id: str | None) -> set[str] | None:
|
||||
"""返回 ``owner_user_id == user_id`` 的自定义技能名集合。
|
||||
|
||||
@@ -110,14 +86,14 @@ def _load_owned_skill_names(user_id: str | None) -> set[str] | None:
|
||||
if not user_id:
|
||||
return None
|
||||
try:
|
||||
from deerflow.persistence.engine import get_session_factory
|
||||
from deerflow.persistence.engine import get_session_factory, run_db_blocking
|
||||
from deerflow.persistence.skills import make_skill_store
|
||||
|
||||
factory = get_session_factory()
|
||||
if factory is None:
|
||||
return None
|
||||
store = make_skill_store(factory)
|
||||
records = _run_coro_blocking(store.list_visible(user_id))
|
||||
records = run_db_blocking(store.list_visible(user_id))
|
||||
return {r["name"] for r in records if r.get("owner_user_id") == user_id}
|
||||
except Exception:
|
||||
logger.debug("Failed to load owned skill names for skill_list filter", exc_info=True)
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -76,6 +77,22 @@ def test_fetch_task_default_kind_is_fetch():
|
||||
assert claimed["action"] is None
|
||||
|
||||
|
||||
def test_claim_next_browser_task_logs_claim_diagnostics(caplog):
|
||||
task_queue.create_browser_task(user_id="u1", url="https://example.com/1")
|
||||
task_queue.create_browser_task(user_id="u2", url="https://example.com/2")
|
||||
|
||||
with caplog.at_level(logging.INFO, logger=task_queue.__name__):
|
||||
claimed = task_queue.claim_next_browser_task(user_id="u1", worker_id="worker-1")
|
||||
|
||||
assert claimed is not None
|
||||
assert "Browser task claim scan" in caplog.text
|
||||
assert "user_id=u1" in caplog.text
|
||||
assert "worker_id=worker-1" in caplog.text
|
||||
assert "total_files=2" in caplog.text
|
||||
assert "matching_user=1" in caplog.text
|
||||
assert "claimed_task_id=" in caplog.text
|
||||
|
||||
|
||||
# --- tool validation --------------------------------------------------------
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user