fix(db): 路由跨 loop 的 DB 访问回引擎主 loop,消除 asyncmy "attached to a different loop"
会商/定时任务/沙箱工具等在 worker 线程的临时 loop 上用 asyncio.run 跑全局引擎连接池的 DB 操作,连接被绑定到随即关闭的临时 loop;池子回收/释放该连接时 asyncmy _terminate_graceful_close 在错误的 loop 上触发 future,持续刷 "got Future attached to a different loop"。 - engine.py: 新增 run_db_blocking(fire_and_forget_db_write 的带返回值版本)。离开主 loop 时用 run_coroutine_threadsafe 把协程送回主 loop 执行、阻塞调用线程拿结果;仅在无主 loop 或正同步处于主 loop 内(送回会死锁)时退回临时 loop。 - thread_paths / prompt / skill_tools: 三处同步→异步阻塞读桥(任务owner解析/常驻技能/ 技能可见性)改走 run_db_blocking,删除各自重复的 _run_coro_blocking。 - llm/tool metrics 中间件: fire-and-forget 写改走 fire_and_forget_db_write。 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
5ecf28e60c
commit
6aff165d09
6 files changed
+147
-96
No files matched your search
@@ -32,6 +32,13 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
_engine: AsyncEngine | None = None
|
||||
_session_factory: async_sessionmaker[AsyncSession] | None = None
|
||||
# The event loop on which the async engine was created. asyncmy/asyncpg connection
|
||||
# pools bind their connections to the loop that opened them; awaiting a write on a
|
||||
# *different* loop reuses those connections cross-loop and raises
|
||||
# "got Future attached to a different loop". We capture the engine's home loop here so
|
||||
# best-effort writes coming from worker threads / other loops can be routed back to it
|
||||
# (see ``fire_and_forget_db_write``).
|
||||
_engine_loop: asyncio.AbstractEventLoop | None = None
|
||||
|
||||
|
||||
def _add_runtime_vendor_path() -> None:
|
||||
@@ -293,12 +300,20 @@ async def init_engine(
|
||||
pool_timeout: Seconds to wait for a free pooled connection before erroring (postgres/mysql).
|
||||
sqlite_dir: Directory to create for SQLite (ensured to exist).
|
||||
"""
|
||||
global _engine, _session_factory
|
||||
global _engine, _session_factory, _engine_loop
|
||||
|
||||
if backend == "memory":
|
||||
logger.info("Persistence backend=memory -- ORM engine not initialized")
|
||||
return
|
||||
|
||||
# Remember the loop that owns the engine's connection pool, so off-loop
|
||||
# best-effort writes (metrics from worker threads, etc.) can be routed here
|
||||
# instead of reusing pooled connections cross-loop.
|
||||
try:
|
||||
_engine_loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
_engine_loop = None
|
||||
|
||||
if backend == "postgres":
|
||||
try:
|
||||
import asyncpg # noqa: F401
|
||||
@@ -459,11 +474,121 @@ def get_engine() -> AsyncEngine | None:
|
||||
return _engine
|
||||
|
||||
|
||||
def get_engine_loop() -> asyncio.AbstractEventLoop | None:
|
||||
"""Return the event loop that owns the engine's connection pool (or None)."""
|
||||
return _engine_loop
|
||||
|
||||
|
||||
def fire_and_forget_db_write(coro) -> None:
|
||||
"""Schedule a best-effort DB write on the engine's home loop, from any loop/thread.
|
||||
|
||||
The async engine's connection pool is bound to the loop that created it
|
||||
(``init_engine``, normally the Gateway's main loop). Running a write on a
|
||||
*different* loop — e.g. a model/tool call that LangGraph executed in a worker
|
||||
thread, which then does ``asyncio.run(store.record(...))`` — reuses those pooled
|
||||
connections cross-loop and raises ``got Future attached to a different loop``.
|
||||
Under concurrency this happens constantly (many calls land in worker threads at
|
||||
once), spamming the log and churning the pool; single-threaded it almost never does.
|
||||
|
||||
This routes every such write back onto the engine's home loop:
|
||||
- already on that loop -> ``create_task`` (in-loop fire-and-forget);
|
||||
- on another loop/thread -> ``run_coroutine_threadsafe`` to the home loop.
|
||||
|
||||
The write itself stays best-effort: if no engine loop is captured (memory backend
|
||||
/ not initialized) the coroutine is closed (avoids a "never awaited" warning) and
|
||||
the call is a no-op. Never raises.
|
||||
"""
|
||||
loop = _engine_loop
|
||||
if loop is None or loop.is_closed():
|
||||
close = getattr(coro, "close", None)
|
||||
if callable(close):
|
||||
close()
|
||||
return
|
||||
try:
|
||||
running = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
running = None
|
||||
try:
|
||||
if running is loop:
|
||||
loop.create_task(coro)
|
||||
else:
|
||||
asyncio.run_coroutine_threadsafe(coro, loop)
|
||||
except Exception: # noqa: BLE001 — best-effort; scheduling must never break the caller
|
||||
close = getattr(coro, "close", None)
|
||||
if callable(close):
|
||||
close()
|
||||
logger.debug("fire_and_forget_db_write: failed to schedule DB write", exc_info=True)
|
||||
|
||||
|
||||
def _run_coro_on_throwaway_loop(coro):
|
||||
"""Last-resort: run *coro* to completion on a fresh event loop.
|
||||
|
||||
Only reached when there is no usable engine home loop, or when the caller is
|
||||
*synchronously* inside the home loop (routing back would deadlock). This
|
||||
touches the shared connection pool from a throwaway loop, so it can still leak
|
||||
a dead-loop-bound connection — but the common worker-thread path no longer
|
||||
gets here (see ``run_db_blocking``).
|
||||
"""
|
||||
try:
|
||||
asyncio.get_running_loop()
|
||||
in_loop = True
|
||||
except RuntimeError:
|
||||
in_loop = False
|
||||
|
||||
if in_loop:
|
||||
import concurrent.futures
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
|
||||
return pool.submit(asyncio.run, coro).result()
|
||||
return asyncio.run(coro)
|
||||
|
||||
|
||||
def run_db_blocking(coro, *, timeout: float | None = 30.0):
|
||||
"""Run an awaited DB coroutine from sync code, safe w.r.t. the engine's home loop.
|
||||
|
||||
The blocking-read sibling of :func:`fire_and_forget_db_write`: where that one
|
||||
drops the result, this returns it — used by the sync→async bridges that must
|
||||
read a row before proceeding (thread-path owner resolution, always-on skills,
|
||||
skill visibility).
|
||||
|
||||
The async engine's connection pool is bound to the loop that created it
|
||||
(``init_engine``). Running a DB coroutine on a *throwaway* loop
|
||||
(``asyncio.run``) checks out pooled connections cross-loop and **leaks** them:
|
||||
the connection ends up bound to a loop that is then closed, and when the pool
|
||||
later recycles/disposes it, asyncmy's ``_terminate_graceful_close`` fires on the
|
||||
wrong loop and raises ``got Future <...> attached to a different loop``. Under
|
||||
concurrency (scheduler / subagent / sandbox-tool worker threads each spinning a
|
||||
fresh loop) this floods the logs.
|
||||
|
||||
So when we are **off** the home loop — the common case, a worker thread with no
|
||||
running loop — route the coroutine onto the home loop and block the *caller*
|
||||
thread for its result. The pool is only ever touched on its own loop, so nothing
|
||||
leaks. Only when there is no home loop, or we are synchronously inside it (where
|
||||
blocking would deadlock), do we fall back to a throwaway loop.
|
||||
|
||||
Never swallows the coroutine's own exception — the sync bridges wrap this call in
|
||||
their own try/except and degrade gracefully.
|
||||
"""
|
||||
loop = _engine_loop
|
||||
try:
|
||||
running = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
running = None
|
||||
if loop is not None and not loop.is_closed() and running is not loop:
|
||||
# Off the home loop (worker thread, or a different loop): execute the coro
|
||||
# on the engine's loop and block this thread for the result. asyncmy's
|
||||
# pooled connections stay on their own loop, so none leak cross-loop.
|
||||
return asyncio.run_coroutine_threadsafe(coro, loop).result(timeout)
|
||||
# No home loop, or we're already synchronously inside it: last resort.
|
||||
return _run_coro_on_throwaway_loop(coro)
|
||||
|
||||
|
||||
async def close_engine() -> None:
|
||||
"""Dispose the engine, release all connections."""
|
||||
global _engine, _session_factory
|
||||
global _engine, _session_factory, _engine_loop
|
||||
if _engine is not None:
|
||||
await _engine.dispose()
|
||||
logger.info("Persistence engine closed")
|
||||
_engine = None
|
||||
_session_factory = None
|
||||
_engine_loop = None
|
||||
Reference in new issue
Block a user