fix: avoid MySQL thread_meta refresh races and harden roundtable bootstrap
Skip post-commit ORM refresh on threads_meta creates so concurrent MySQL read/write-split deployments no longer fail with Could not refresh instance; clean up partial roundtable threads on checkpoint failure, and ship related position-roundtable/frontend build fixes. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
e52261fc3d
commit
a9d66536c3
12 files changed
+503
-127
No files matched your search
@@ -127,7 +127,25 @@ Clients may retry the compatibility route only for a framework/proxy-level
|
||||
`multi_agent._create_thread_direct`, exactly like multi-agent initialization;
|
||||
do not restore the old HTTP loopback to `/api/threads`, because deployed
|
||||
Gateway ports/proxies may make the automatic page initialization fail while
|
||||
later manual turns still work. Streaming stays on the existing loopback path.
|
||||
later manual turns still work. If checkpoint creation fails or the parallel seat
|
||||
task is cancelled, the helper compensates by deleting any partial checkpoint and
|
||||
metadata row (the cancellation branch runs under `asyncio.shield` so a second
|
||||
cancellation cannot abort it half-way). Streaming stays on the existing loopback
|
||||
path.
|
||||
|
||||
`ThreadMetaStore.create` never issues a post-commit `session.refresh()`. Every
|
||||
`ThreadMetaRow` column is assigned before commit and the session factory uses
|
||||
`expire_on_commit=False`, so the returned dict is built from the in-memory row.
|
||||
The removed read-back ran on a *different* pooled connection than the INSERT,
|
||||
which a MySQL read/write-splitting endpoint can route to a lagging replica — the
|
||||
row comes back empty and SQLAlchemy raises `Could not refresh instance`, surfacing
|
||||
as `500 "Failed to create thread"`. Concurrency makes that far more likely, so a
|
||||
single-user deployment never sees it. No caller uses the return value, and all
|
||||
seven call sites (`multi_agent._create_thread_direct`, `services.start_run`,
|
||||
`routers/threads.create_thread`, `roundtable_inprocess_gateway`,
|
||||
`scheduler/service`, `thread_shares`, `authz`) benefit. Keep new `create`
|
||||
implementations read-back-free; `tests/test_position_roundtable_sessions.py::
|
||||
test_thread_meta_create_never_reads_back_after_commit` guards this.
|
||||
|
||||
### Harness / App Split
|
||||
|
||||
|
||||
@@ -481,8 +481,14 @@ Its live intent conversation still uses the shared `/api/intent/init` and
|
||||
`/api/intent/stream` endpoints used by the original multi-agent roundtable.
|
||||
`/api/intent/init` creates its empty thread directly through the in-process
|
||||
thread store/checkpointer, matching multi-agent initialization and avoiding a
|
||||
deployment-sensitive `127.0.0.1:<gateway-port>` loopback. Only the recovery
|
||||
snapshot layer is position-specific. Deployments must expose
|
||||
deployment-sensitive `127.0.0.1:<gateway-port>` loopback. If initial checkpoint
|
||||
creation fails or is cancelled, bootstrap removes both the partial checkpoint and
|
||||
its thread metadata row. Thread metadata creation issues no post-commit ORM
|
||||
read-back on any code path: all returned fields are assigned before commit, and
|
||||
skipping the extra read keeps a MySQL read/write-splitting endpoint from sending
|
||||
it to a lagging replica (which fails the request with
|
||||
`Could not refresh instance`). Only the
|
||||
recovery snapshot layer is position-specific. Deployments must expose
|
||||
either `/api/position-roundtable/*` or the compatibility mirror
|
||||
`/api/multi-agent/position-roundtable/*`; updated clients probe both and only
|
||||
retry when the first response is the framework/proxy-level `404 Not Found`.
|
||||
|
||||
@@ -442,10 +442,30 @@ async def _create_thread_direct(request: Request, *, metadata: dict[str, Any] |
|
||||
"parents": {},
|
||||
"created_at": now,
|
||||
}
|
||||
|
||||
async def _cleanup_partial_thread() -> None:
|
||||
delete_checkpoint = getattr(checkpointer, "adelete_thread", None)
|
||||
if callable(delete_checkpoint):
|
||||
try:
|
||||
await delete_checkpoint(thread_id)
|
||||
except Exception:
|
||||
logger.exception("[init-direct] partial checkpoint cleanup failed for %s", thread_id)
|
||||
try:
|
||||
await thread_store.delete(thread_id)
|
||||
except Exception:
|
||||
logger.exception("[init-direct] partial thread_meta cleanup failed for %s", thread_id)
|
||||
|
||||
try:
|
||||
await checkpointer.aput(config, empty_checkpoint(), ckpt_metadata, {})
|
||||
except asyncio.CancelledError:
|
||||
# A sibling seat may fail while this task is between metadata commit and
|
||||
# checkpoint creation. Compensate before propagating cancellation, under
|
||||
# shield so a second cancellation cannot abort the cleanup half-way.
|
||||
await asyncio.shield(_cleanup_partial_thread())
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception("[init-direct] checkpoint create failed for %s", thread_id)
|
||||
await _cleanup_partial_thread()
|
||||
raise HTTPException(status_code=500, detail="Failed to create thread")
|
||||
return thread_id
|
||||
|
||||
|
||||
+6
@@ -30,6 +30,12 @@ class ThreadMetaStore(abc.ABC):
|
||||
display_name: str | None = None,
|
||||
metadata: dict | None = None,
|
||||
) -> dict:
|
||||
"""Create one metadata row.
|
||||
|
||||
Implementations must assemble the returned dict from the values passed
|
||||
in, without a post-commit read-back: on MySQL such a read can be routed
|
||||
to a lagging replica behind a read/write-splitting endpoint and fail.
|
||||
"""
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
|
||||
+9
-1
@@ -45,6 +45,7 @@ class ThreadMetaRepository(ThreadMetaStore):
|
||||
assistant_id=assistant_id,
|
||||
user_id=resolved_user_id,
|
||||
display_name=display_name,
|
||||
status="idle",
|
||||
metadata_json=metadata or {},
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
@@ -52,7 +53,14 @@ class ThreadMetaRepository(ThreadMetaStore):
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
# Deliberately no session.refresh(): every column above is assigned
|
||||
# explicitly and the session factory uses expire_on_commit=False, so
|
||||
# the in-memory row is already complete. The re-SELECT would run on a
|
||||
# different pooled connection than the INSERT, which a MySQL
|
||||
# read/write-splitting endpoint can route to a lagging replica --
|
||||
# the row comes back empty and SQLAlchemy raises
|
||||
# "Could not refresh instance". Concurrency makes that far more
|
||||
# likely, so a single-user deployment never sees it.
|
||||
return self._row_to_dict(row)
|
||||
|
||||
async def get(
|
||||
|
||||
@@ -2,19 +2,24 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from fastapi import HTTPException
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from app.gateway import deps as gateway_deps
|
||||
from app.gateway import services as gateway_services
|
||||
from app.gateway.app import create_app
|
||||
from app.gateway.routers import intent as intent_router
|
||||
from app.gateway.routers import multi_agent
|
||||
from app.gateway.routers import position_roundtable as position_roundtable_router
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.position_roundtable import PositionRoundtableRepository
|
||||
from deerflow.persistence.thread_meta.sql import ThreadMetaRepository
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
@@ -971,3 +976,120 @@ async def test_locked_node_cannot_bind_a_thread_before_its_stage_is_ready():
|
||||
)
|
||||
assert error.value.status_code == 409
|
||||
assert "未解锁" in str(error.value.detail)
|
||||
|
||||
|
||||
class _ThreadMetaSessionProbe:
|
||||
def __init__(self, events: list[str]) -> None:
|
||||
self._events = events
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc, tb) -> None:
|
||||
return None
|
||||
|
||||
def add(self, row) -> None:
|
||||
self._events.append(f"add:{row.thread_id}")
|
||||
|
||||
async def commit(self) -> None:
|
||||
self._events.append("commit")
|
||||
|
||||
async def refresh(self, row) -> None:
|
||||
self._events.append(f"refresh:{row.thread_id}")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_thread_meta_create_never_reads_back_after_commit():
|
||||
"""No caller may depend on a post-commit re-SELECT of the new row.
|
||||
|
||||
On MySQL behind a read/write-splitting endpoint that read can land on a
|
||||
lagging replica, come back empty, and raise "Could not refresh instance".
|
||||
"""
|
||||
|
||||
events: list[str] = []
|
||||
repo = ThreadMetaRepository(lambda: _ThreadMetaSessionProbe(events)) # type: ignore[arg-type]
|
||||
|
||||
direct = await repo.create(
|
||||
"roundtable-thread",
|
||||
user_id=None,
|
||||
metadata={"thread_type": "roundtable"},
|
||||
)
|
||||
ordinary = await repo.create("ordinary-thread", user_id=None, display_name="对话")
|
||||
|
||||
assert events == [
|
||||
"add:roundtable-thread",
|
||||
"commit",
|
||||
"add:ordinary-thread",
|
||||
"commit",
|
||||
]
|
||||
# The returned dict is assembled from the in-memory row, so it must still be
|
||||
# complete for every column -- including ones no caller passed in.
|
||||
assert direct["status"] == "idle"
|
||||
assert direct["metadata"] == {"thread_type": "roundtable"}
|
||||
assert direct["created_at"] and direct["updated_at"]
|
||||
assert ordinary["status"] == "idle"
|
||||
assert ordinary["display_name"] == "对话"
|
||||
|
||||
|
||||
class _ThreadStoreProbe:
|
||||
def __init__(self) -> None:
|
||||
self.created: list[tuple[str, dict[str, Any]]] = []
|
||||
self.deleted: list[str] = []
|
||||
|
||||
async def create(self, thread_id: str, **kwargs) -> dict[str, Any]:
|
||||
self.created.append((thread_id, kwargs))
|
||||
return {"thread_id": thread_id}
|
||||
|
||||
async def delete(self, thread_id: str) -> None:
|
||||
self.deleted.append(thread_id)
|
||||
|
||||
|
||||
class _FailingCheckpointer:
|
||||
def __init__(self, exc: BaseException) -> None:
|
||||
self._exc = exc
|
||||
self.deleted: list[str] = []
|
||||
|
||||
async def aput(self, *args, **kwargs):
|
||||
raise self._exc
|
||||
|
||||
async def adelete_thread(self, thread_id: str) -> None:
|
||||
self.deleted.append(thread_id)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_direct_thread_creation_cleans_failed_checkpoint(monkeypatch):
|
||||
store = _ThreadStoreProbe()
|
||||
checkpointer = _FailingCheckpointer(RuntimeError("checkpoint unavailable"))
|
||||
monkeypatch.setattr(gateway_deps, "get_thread_store", lambda request: store)
|
||||
monkeypatch.setattr(gateway_deps, "get_checkpointer", lambda request: checkpointer)
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await multi_agent._create_thread_direct(
|
||||
SimpleNamespace(),
|
||||
metadata={"thread_type": "roundtable"},
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 500
|
||||
assert len(store.created) == 1
|
||||
thread_id, create_kwargs = store.created[0]
|
||||
assert create_kwargs["metadata"] == {"thread_type": "roundtable"}
|
||||
assert store.deleted == [thread_id]
|
||||
assert checkpointer.deleted == [thread_id]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_direct_thread_creation_cleans_metadata_when_cancelled(monkeypatch):
|
||||
store = _ThreadStoreProbe()
|
||||
checkpointer = _FailingCheckpointer(asyncio.CancelledError())
|
||||
monkeypatch.setattr(gateway_deps, "get_thread_store", lambda request: store)
|
||||
monkeypatch.setattr(gateway_deps, "get_checkpointer", lambda request: checkpointer)
|
||||
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await multi_agent._create_thread_direct(
|
||||
SimpleNamespace(),
|
||||
metadata={"thread_type": "roundtable"},
|
||||
)
|
||||
|
||||
thread_id, _ = store.created[0]
|
||||
assert store.deleted == [thread_id]
|
||||
assert checkpointer.deleted == [thread_id]
|
||||
Reference in new issue
Block a user