fix: avoid MySQL thread_meta refresh races and harden roundtable bootstrap

Skip post-commit ORM refresh on threads_meta creates so concurrent MySQL
read/write-split deployments no longer fail with Could not refresh instance;
clean up partial roundtable threads on checkpoint failure, and ship related
position-roundtable/frontend build fixes.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
董宏凯andCursor committed 2026-07-31 21:00:56 +08:00
1 parent e52261fc3d
commit a9d66536c3
12 files changed
+503 -127

No files matched your search

+19 -1
View File
@@ -127,7 +127,25 @@ Clients may retry the compatibility route only for a framework/proxy-level
`multi_agent._create_thread_direct`, exactly like multi-agent initialization;
do not restore the old HTTP loopback to `/api/threads`, because deployed
Gateway ports/proxies may make the automatic page initialization fail while
later manual turns still work. Streaming stays on the existing loopback path.
later manual turns still work. If checkpoint creation fails or the parallel seat
task is cancelled, the helper compensates by deleting any partial checkpoint and
metadata row (the cancellation branch runs under `asyncio.shield` so a second
cancellation cannot abort it half-way). Streaming stays on the existing loopback
path.
`ThreadMetaStore.create` never issues a post-commit `session.refresh()`. Every
`ThreadMetaRow` column is assigned before commit and the session factory uses
`expire_on_commit=False`, so the returned dict is built from the in-memory row.
The removed read-back ran on a *different* pooled connection than the INSERT,
which a MySQL read/write-splitting endpoint can route to a lagging replica — the
row comes back empty and SQLAlchemy raises `Could not refresh instance`, surfacing
as `500 "Failed to create thread"`. Concurrency makes that far more likely, so a
single-user deployment never sees it. No caller uses the return value, and all
seven call sites (`multi_agent._create_thread_direct`, `services.start_run`,
`routers/threads.create_thread`, `roundtable_inprocess_gateway`,
`scheduler/service`, `thread_shares`, `authz`) benefit. Keep new `create`
implementations read-back-free; `tests/test_position_roundtable_sessions.py::
test_thread_meta_create_never_reads_back_after_commit` guards this.
### Harness / App Split
+8 -2
View File
@@ -481,8 +481,14 @@ Its live intent conversation still uses the shared `/api/intent/init` and
`/api/intent/stream` endpoints used by the original multi-agent roundtable.
`/api/intent/init` creates its empty thread directly through the in-process
thread store/checkpointer, matching multi-agent initialization and avoiding a
deployment-sensitive `127.0.0.1:<gateway-port>` loopback. Only the recovery
snapshot layer is position-specific. Deployments must expose
deployment-sensitive `127.0.0.1:<gateway-port>` loopback. If initial checkpoint
creation fails or is cancelled, bootstrap removes both the partial checkpoint and
its thread metadata row. Thread metadata creation issues no post-commit ORM
read-back on any code path: all returned fields are assigned before commit, and
skipping the extra read keeps a MySQL read/write-splitting endpoint from sending
it to a lagging replica (which fails the request with
`Could not refresh instance`). Only the
recovery snapshot layer is position-specific. Deployments must expose
either `/api/position-roundtable/*` or the compatibility mirror
`/api/multi-agent/position-roundtable/*`; updated clients probe both and only
retry when the first response is the framework/proxy-level `404 Not Found`.
@@ -442,10 +442,30 @@ async def _create_thread_direct(request: Request, *, metadata: dict[str, Any] |
"parents": {},
"created_at": now,
}
async def _cleanup_partial_thread() -> None:
delete_checkpoint = getattr(checkpointer, "adelete_thread", None)
if callable(delete_checkpoint):
try:
await delete_checkpoint(thread_id)
except Exception:
logger.exception("[init-direct] partial checkpoint cleanup failed for %s", thread_id)
try:
await thread_store.delete(thread_id)
except Exception:
logger.exception("[init-direct] partial thread_meta cleanup failed for %s", thread_id)
try:
await checkpointer.aput(config, empty_checkpoint(), ckpt_metadata, {})
except asyncio.CancelledError:
# A sibling seat may fail while this task is between metadata commit and
# checkpoint creation. Compensate before propagating cancellation, under
# shield so a second cancellation cannot abort the cleanup half-way.
await asyncio.shield(_cleanup_partial_thread())
raise
except Exception:
logger.exception("[init-direct] checkpoint create failed for %s", thread_id)
await _cleanup_partial_thread()
raise HTTPException(status_code=500, detail="Failed to create thread")
return thread_id
@@ -30,6 +30,12 @@ class ThreadMetaStore(abc.ABC):
display_name: str | None = None,
metadata: dict | None = None,
) -> dict:
"""Create one metadata row.
Implementations must assemble the returned dict from the values passed
in, without a post-commit read-back: on MySQL such a read can be routed
to a lagging replica behind a read/write-splitting endpoint and fail.
"""
pass
@abc.abstractmethod
@@ -45,6 +45,7 @@ class ThreadMetaRepository(ThreadMetaStore):
assistant_id=assistant_id,
user_id=resolved_user_id,
display_name=display_name,
status="idle",
metadata_json=metadata or {},
created_at=now,
updated_at=now,
@@ -52,7 +53,14 @@ class ThreadMetaRepository(ThreadMetaStore):
async with self._sf() as session:
session.add(row)
await session.commit()
await session.refresh(row)
# Deliberately no session.refresh(): every column above is assigned
# explicitly and the session factory uses expire_on_commit=False, so
# the in-memory row is already complete. The re-SELECT would run on a
# different pooled connection than the INSERT, which a MySQL
# read/write-splitting endpoint can route to a lagging replica --
# the row comes back empty and SQLAlchemy raises
# "Could not refresh instance". Concurrency makes that far more
# likely, so a single-user deployment never sees it.
return self._row_to_dict(row)
async def get(
@@ -2,19 +2,24 @@
from __future__ import annotations
import asyncio
from types import SimpleNamespace
from typing import Any
import pytest
import pytest_asyncio
from fastapi import HTTPException
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from app.gateway import deps as gateway_deps
from app.gateway import services as gateway_services
from app.gateway.app import create_app
from app.gateway.routers import intent as intent_router
from app.gateway.routers import multi_agent
from app.gateway.routers import position_roundtable as position_roundtable_router
from deerflow.persistence.base import Base
from deerflow.persistence.position_roundtable import PositionRoundtableRepository
from deerflow.persistence.thread_meta.sql import ThreadMetaRepository
@pytest_asyncio.fixture
@@ -971,3 +976,120 @@ async def test_locked_node_cannot_bind_a_thread_before_its_stage_is_ready():
)
assert error.value.status_code == 409
assert "未解锁" in str(error.value.detail)
class _ThreadMetaSessionProbe:
def __init__(self, events: list[str]) -> None:
self._events = events
async def __aenter__(self):
return self
async def __aexit__(self, exc_type, exc, tb) -> None:
return None
def add(self, row) -> None:
self._events.append(f"add:{row.thread_id}")
async def commit(self) -> None:
self._events.append("commit")
async def refresh(self, row) -> None:
self._events.append(f"refresh:{row.thread_id}")
@pytest.mark.asyncio
async def test_thread_meta_create_never_reads_back_after_commit():
"""No caller may depend on a post-commit re-SELECT of the new row.
On MySQL behind a read/write-splitting endpoint that read can land on a
lagging replica, come back empty, and raise "Could not refresh instance".
"""
events: list[str] = []
repo = ThreadMetaRepository(lambda: _ThreadMetaSessionProbe(events)) # type: ignore[arg-type]
direct = await repo.create(
"roundtable-thread",
user_id=None,
metadata={"thread_type": "roundtable"},
)
ordinary = await repo.create("ordinary-thread", user_id=None, display_name="对话")
assert events == [
"add:roundtable-thread",
"commit",
"add:ordinary-thread",
"commit",
]
# The returned dict is assembled from the in-memory row, so it must still be
# complete for every column -- including ones no caller passed in.
assert direct["status"] == "idle"
assert direct["metadata"] == {"thread_type": "roundtable"}
assert direct["created_at"] and direct["updated_at"]
assert ordinary["status"] == "idle"
assert ordinary["display_name"] == "对话"
class _ThreadStoreProbe:
def __init__(self) -> None:
self.created: list[tuple[str, dict[str, Any]]] = []
self.deleted: list[str] = []
async def create(self, thread_id: str, **kwargs) -> dict[str, Any]:
self.created.append((thread_id, kwargs))
return {"thread_id": thread_id}
async def delete(self, thread_id: str) -> None:
self.deleted.append(thread_id)
class _FailingCheckpointer:
def __init__(self, exc: BaseException) -> None:
self._exc = exc
self.deleted: list[str] = []
async def aput(self, *args, **kwargs):
raise self._exc
async def adelete_thread(self, thread_id: str) -> None:
self.deleted.append(thread_id)
@pytest.mark.asyncio
async def test_direct_thread_creation_cleans_failed_checkpoint(monkeypatch):
store = _ThreadStoreProbe()
checkpointer = _FailingCheckpointer(RuntimeError("checkpoint unavailable"))
monkeypatch.setattr(gateway_deps, "get_thread_store", lambda request: store)
monkeypatch.setattr(gateway_deps, "get_checkpointer", lambda request: checkpointer)
with pytest.raises(HTTPException) as exc_info:
await multi_agent._create_thread_direct(
SimpleNamespace(),
metadata={"thread_type": "roundtable"},
)
assert exc_info.value.status_code == 500
assert len(store.created) == 1
thread_id, create_kwargs = store.created[0]
assert create_kwargs["metadata"] == {"thread_type": "roundtable"}
assert store.deleted == [thread_id]
assert checkpointer.deleted == [thread_id]
@pytest.mark.asyncio
async def test_direct_thread_creation_cleans_metadata_when_cancelled(monkeypatch):
store = _ThreadStoreProbe()
checkpointer = _FailingCheckpointer(asyncio.CancelledError())
monkeypatch.setattr(gateway_deps, "get_thread_store", lambda request: store)
monkeypatch.setattr(gateway_deps, "get_checkpointer", lambda request: checkpointer)
with pytest.raises(asyncio.CancelledError):
await multi_agent._create_thread_direct(
SimpleNamespace(),
metadata={"thread_type": "roundtable"},
)
thread_id, _ = store.created[0]
assert store.deleted == [thread_id]
assert checkpointer.deleted == [thread_id]