代码提交
This commit is contained in:
1 parent
d93cc62806
commit
ddf83f7728
25 files changed
+4086
-424
No files matched your search
@@ -12,7 +12,7 @@ import asyncio
|
||||
import logging
|
||||
from collections.abc import AsyncGenerator, Callable
|
||||
from contextlib import AsyncExitStack, asynccontextmanager
|
||||
from typing import TYPE_CHECKING, TypeVar, cast
|
||||
from typing import TYPE_CHECKING, Any, TypeVar, cast
|
||||
|
||||
from fastapi import FastAPI, HTTPException, Request
|
||||
from langgraph.types import Checkpointer
|
||||
@@ -39,6 +39,67 @@ if TYPE_CHECKING:
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
def get_gateway_worker_topology() -> dict[str, Any]:
|
||||
"""Detect the current worker count and source.
|
||||
|
||||
Returns ``{"worker_count": int|None, "source": str, "is_multi_worker": bool}``.
|
||||
Sources checked in order: ``WEB_CONCURRENCY`` env var, ``/proc`` scan (Linux).
|
||||
"""
|
||||
import os as _os
|
||||
|
||||
wc = _os.environ.get("WEB_CONCURRENCY", "").strip()
|
||||
if wc and wc.isdigit():
|
||||
count = int(wc)
|
||||
return {"worker_count": count, "source": "WEB_CONCURRENCY", "is_multi_worker": count > 1}
|
||||
|
||||
if not _os.name == "posix":
|
||||
return {"worker_count": None, "source": "unknown", "is_multi_worker": False}
|
||||
|
||||
try:
|
||||
from pathlib import Path as _Path
|
||||
|
||||
peers = 0
|
||||
proc = _Path("/proc")
|
||||
if proc.exists():
|
||||
for entry in proc.iterdir():
|
||||
if not entry.name.isdigit():
|
||||
continue
|
||||
try:
|
||||
cmdline = (entry / "cmdline").read_bytes().replace(b"\x00", b" ").decode("utf-8", "ignore")
|
||||
except OSError:
|
||||
continue
|
||||
if "uvicorn" in cmdline and "app.gateway.app" in cmdline:
|
||||
peers += 1
|
||||
if peers > 0:
|
||||
return {"worker_count": peers, "source": "process_scan", "is_multi_worker": peers > 1}
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
return {"worker_count": None, "source": "unknown", "is_multi_worker": False}
|
||||
|
||||
|
||||
async def _background_reconcile_loop(service: Any, *, interval: int) -> None:
|
||||
"""Periodically reconcile active conversation runs."""
|
||||
while True:
|
||||
try:
|
||||
await asyncio.sleep(interval)
|
||||
await service.reconcile_active_conversations()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception: # noqa: BLE001
|
||||
logger.warning("[maf-roundtable] background reconcile failed", exc_info=True)
|
||||
|
||||
|
||||
async def _startup_reconcile(service: Any) -> None:
|
||||
"""One-shot reconcile of all active runs at gateway startup."""
|
||||
try:
|
||||
written = await service.reconcile_active_conversations()
|
||||
if written:
|
||||
logger.info("[maf-roundtable] startup reconcile wrote %d conversation events", written)
|
||||
except Exception: # noqa: BLE001
|
||||
logger.warning("[maf-roundtable] startup reconcile failed", exc_info=True)
|
||||
|
||||
|
||||
def resolve_roundtable_gateway_mode(raw: str | None) -> str:
|
||||
"""解析 ``ROUNDTABLE_JOB_GATEWAY``:**默认 ``inprocess``(真模型后台挂起)**。
|
||||
|
||||
@@ -321,11 +382,24 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
|
||||
app.state.maf_roundtable_store = make_maf_roundtable_store(sf)
|
||||
maf_enabled = False
|
||||
maf_dispatch_ready = False
|
||||
maf_dispatch_status = "configuration_unavailable"
|
||||
try:
|
||||
from app.maf_roundtable import MafRoundtableService
|
||||
|
||||
maf_cfg = getattr(config, "maf_roundtable", None)
|
||||
maf_enabled = bool(getattr(maf_cfg, "enabled", False)) if maf_cfg is not None else False
|
||||
maf_cfg = getattr(config, "maf_roundtable", None)
|
||||
maf_enabled = bool(getattr(maf_cfg, "enabled", False)) if maf_cfg is not None else False
|
||||
if maf_cfg is not None:
|
||||
from app.maf_roundtable.dispatch_contracts import MafDispatchCatalog
|
||||
|
||||
maf_dispatch_ready, maf_dispatch_status = MafDispatchCatalog(maf_cfg).readiness()
|
||||
if maf_enabled and not maf_dispatch_ready:
|
||||
logger.error(
|
||||
"[maf-roundtable] enabled but no trusted manager route is configured; "
|
||||
"forcing the conversation entry OFF (%s)",
|
||||
maf_dispatch_status,
|
||||
)
|
||||
maf_enabled = False
|
||||
maf_admin_diag = (
|
||||
bool(getattr(maf_cfg, "admin_diagnostic_access", True)) if maf_cfg is not None else True
|
||||
)
|
||||
@@ -348,7 +422,8 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
"forcing the feature OFF: %s",
|
||||
cfg_exc,
|
||||
)
|
||||
maf_enabled = False
|
||||
maf_enabled = False
|
||||
maf_dispatch_ready = False
|
||||
if app.state.maf_roundtable_store is not None:
|
||||
app.state.maf_roundtable_service = MafRoundtableService(
|
||||
app.state.maf_roundtable_store,
|
||||
@@ -371,24 +446,48 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
app.state.maf_roundtable_service = None
|
||||
|
||||
# ── MAF roundtable V2 Conversation service + SSE event hub ──
|
||||
# P0-2: multi-worker guard — memory hub cannot broadcast across
|
||||
# workers. When detected, disable the conversation SSE write path
|
||||
# (reads stay open for admin history review).
|
||||
_topology = get_gateway_worker_topology()
|
||||
_maf_cfg = getattr(config, "maf_roundtable", None)
|
||||
_conv_backend = getattr(_maf_cfg, "conversation_event_backend", "memory")
|
||||
_require_single = getattr(_maf_cfg, "require_single_worker_for_conversation_sse", True)
|
||||
_stream_blocked = (
|
||||
_conv_backend == "memory"
|
||||
and _topology["is_multi_worker"]
|
||||
and _require_single
|
||||
)
|
||||
if _stream_blocked:
|
||||
logger.error(
|
||||
"[maf-roundtable] conversation_event_backend=memory with %d workers "
|
||||
"(%s) — SSE stream disabled. Set WEB_CONCURRENCY=1 or deploy Redis "
|
||||
"Pub/Sub before enabling multi-worker conversation SSE.",
|
||||
_topology["worker_count"],
|
||||
_topology["source"],
|
||||
)
|
||||
|
||||
try:
|
||||
from app.maf_roundtable.conversation_event_hub import ConversationEventHub
|
||||
|
||||
app.state.maf_conversation_event_hub = ConversationEventHub()
|
||||
if _stream_blocked:
|
||||
app.state.maf_conversation_event_hub = None
|
||||
else:
|
||||
app.state.maf_conversation_event_hub = ConversationEventHub()
|
||||
|
||||
if app.state.maf_roundtable_store is not None:
|
||||
from app.maf_roundtable.conversation_service import MafConversationService
|
||||
from app.maf_roundtable.dispatcher import get_dispatcher
|
||||
|
||||
app.state.maf_conversation_service = MafConversationService(
|
||||
app.state.maf_roundtable_store,
|
||||
maf_config=getattr(config, "maf_roundtable", None),
|
||||
event_hub=app.state.maf_conversation_event_hub,
|
||||
agent_store=getattr(app.state, "agent_store", None),
|
||||
app_config=config,
|
||||
dispatcher=get_dispatcher(app),
|
||||
enabled=maf_enabled,
|
||||
)
|
||||
if app.state.maf_roundtable_store is not None:
|
||||
from app.maf_roundtable.conversation_service import MafConversationService
|
||||
from app.maf_roundtable.dispatcher import get_dispatcher
|
||||
|
||||
app.state.maf_conversation_service = MafConversationService(
|
||||
app.state.maf_roundtable_store,
|
||||
maf_config=_maf_cfg,
|
||||
event_hub=app.state.maf_conversation_event_hub,
|
||||
agent_store=getattr(app.state, "agent_store", None),
|
||||
app_config=config,
|
||||
dispatcher=get_dispatcher(app),
|
||||
enabled=maf_enabled,
|
||||
)
|
||||
else:
|
||||
app.state.maf_conversation_service = None
|
||||
except Exception: # noqa: BLE001
|
||||
@@ -396,6 +495,36 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
app.state.maf_conversation_service = None
|
||||
app.state.maf_conversation_event_hub = None
|
||||
|
||||
app.state.maf_conversation_dispatch_ready = bool(maf_enabled and maf_dispatch_ready)
|
||||
app.state.maf_conversation_dispatch_status = maf_dispatch_status
|
||||
app.state.maf_conversation_stream_ready = not _stream_blocked and app.state.maf_conversation_event_hub is not None
|
||||
app.state.maf_conversation_stream_status = (
|
||||
"multi_worker_memory_hub_unsupported" if _stream_blocked
|
||||
else "ok" if app.state.maf_conversation_event_hub is not None
|
||||
else "hub_not_initialized"
|
||||
)
|
||||
app.state.maf_conversation_event_backend = _conv_backend
|
||||
|
||||
# ── Background reconcile task for conversation event projection ──
|
||||
_reconcile_task: asyncio.Task | None = None
|
||||
if (
|
||||
app.state.maf_conversation_service is not None
|
||||
and maf_enabled
|
||||
and not _stream_blocked
|
||||
):
|
||||
_reconcile_interval = getattr(_maf_cfg, "background_reconcile_interval_seconds", 10)
|
||||
_reconcile_task = asyncio.create_task(
|
||||
_background_reconcile_loop(
|
||||
app.state.maf_conversation_service,
|
||||
interval=max(5, int(_reconcile_interval)),
|
||||
)
|
||||
)
|
||||
app.state.maf_conversation_reconcile_task = _reconcile_task
|
||||
logger.info("[maf-roundtable] background reconcile started (interval=%ss)", _reconcile_interval)
|
||||
|
||||
# Startup reconcile: scan active/recent runs once
|
||||
asyncio.create_task(_startup_reconcile(app.state.maf_conversation_service))
|
||||
|
||||
from deerflow.persistence.roundtable_jobs import (
|
||||
make_roundtable_job_store,
|
||||
)
|
||||
@@ -485,14 +614,22 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
# P1: shut down the SSE event hub before closing the DB engine
|
||||
# so active SSE streams are released cleanly.
|
||||
reconcile_task = getattr(app.state, "maf_conversation_reconcile_task", None)
|
||||
if reconcile_task is not None:
|
||||
reconcile_task.cancel()
|
||||
try:
|
||||
await reconcile_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception: # noqa: BLE001
|
||||
logger.warning("[maf-roundtable] reconcile task shutdown error", exc_info=True)
|
||||
|
||||
hub = getattr(app.state, "maf_conversation_event_hub", None)
|
||||
if hub is not None:
|
||||
try:
|
||||
hub.shutdown()
|
||||
await hub.aclose()
|
||||
except Exception: # noqa: BLE001
|
||||
logger.warning("[maf-roundtable] hub shutdown failed", exc_info=True)
|
||||
logger.warning("[maf-roundtable] hub aclose failed", exc_info=True)
|
||||
|
||||
from deerflow.persistence.notifications import set_notification_store
|
||||
|
||||
|
||||
@@ -82,8 +82,18 @@ def _user_ctx(request: Request) -> tuple[str, bool]:
|
||||
# ── REST endpoints (Phase 1) ──────────────────────────────────────────────
|
||||
|
||||
|
||||
def _check_stream_ready(request: Request) -> None:
|
||||
stream_ready = getattr(request.app.state, "maf_conversation_stream_ready", True)
|
||||
if not stream_ready:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail=getattr(request.app.state, "maf_conversation_stream_status", "stream_not_ready"),
|
||||
)
|
||||
|
||||
|
||||
@router.post("/conversations", response_model=ConversationResponse, status_code=status.HTTP_201_CREATED)
|
||||
async def create_conversation(body: CreateConversationRequest, request: Request):
|
||||
_check_stream_ready(request)
|
||||
svc = _get_conversation_service(request)
|
||||
user_id, _ = _user_ctx(request)
|
||||
try:
|
||||
@@ -142,6 +152,7 @@ async def list_messages(
|
||||
|
||||
@router.post("/conversations/{conversation_id}/messages", response_model=SendMessageResponse, status_code=status.HTTP_201_CREATED)
|
||||
async def send_message(conversation_id: str, body: SendMessageRequest, request: Request):
|
||||
_check_stream_ready(request)
|
||||
svc = _get_conversation_service(request)
|
||||
user_id, _ = _user_ctx(request)
|
||||
try:
|
||||
@@ -198,53 +209,44 @@ def _is_terminal(event: dict[str, Any]) -> bool:
|
||||
return event.get("type") in _TERMINAL_EVENT_TYPES
|
||||
|
||||
|
||||
@router.get("/conversations/{conversation_id}/stream")
|
||||
async def stream_conversation(
|
||||
conversation_id: str,
|
||||
def build_conversation_event_generator(
|
||||
*,
|
||||
service: MafConversationService,
|
||||
hub: ConversationEventHub,
|
||||
request: Request,
|
||||
after_seq: int = Query(default=0, ge=0),
|
||||
):
|
||||
"""SSE stream for one conversation (§4.2).
|
||||
conversation_id: str,
|
||||
after_seq: int = 0,
|
||||
) -> Any:
|
||||
"""Extracted async generator for SSE streaming — testable without HTTP.
|
||||
|
||||
Single fetch-reader connection per conversation. The ordering is:
|
||||
The ordering is:
|
||||
|
||||
1. **Open subscription** (register before any DB query — P0-1 fix).
|
||||
2. **Replay** from DB: ``list_events(after_seq)``.
|
||||
3. **Stabilise**: re-query while the hub generation advanced during replay.
|
||||
4. **Snapshot** marker.
|
||||
5. **Real-time**: ``wait_for_change`` → query DB → yield events.
|
||||
1. **Reconcile** any missed projections for the active run (P0-1 auto).
|
||||
2. **Open subscription** (register before any DB query — P0-1 fix).
|
||||
3. **Replay** from DB: ``list_events(after_seq)``.
|
||||
4. **Stabilise**: re-query while the hub generation advanced during replay.
|
||||
5. **Snapshot** marker.
|
||||
6. **Real-time**: ``wait_for_change`` → query DB → yield events.
|
||||
|
||||
On terminal events (``run.completed`` / ``run.error`` / ``run.cancelled``)
|
||||
the stream sends the event and closes (P0-2 fix).
|
||||
|
||||
Heartbeat every 15 s. Disconnect detection via ``request.is_disconnected()``.
|
||||
On terminal events the stream sends the event and closes (P0-2 fix).
|
||||
Heartbeat every ``hub.heartbeat_interval`` seconds.
|
||||
"""
|
||||
svc = _get_conversation_service(request)
|
||||
hub = _get_event_hub(request)
|
||||
user_id, is_admin = _user_ctx(request)
|
||||
store = service._store # noqa: SLF001
|
||||
|
||||
if hub is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail="SSE event hub not configured",
|
||||
)
|
||||
async def _gen():
|
||||
try:
|
||||
active_run_id = await _get_active_run_id(store, conversation_id)
|
||||
if active_run_id:
|
||||
try:
|
||||
await service.reconcile_conversation_events(run_id=active_run_id)
|
||||
except Exception: # noqa: BLE001
|
||||
logger.warning("SSE connect reconcile failed conv=%s", conversation_id, exc_info=True)
|
||||
except Exception: # noqa: BLE001
|
||||
logger.warning("SSE connect reconcile lookup failed conv=%s", conversation_id, exc_info=True)
|
||||
|
||||
# Ownership check before opening the stream.
|
||||
try:
|
||||
await svc.get_conversation(
|
||||
conversation_id=conversation_id, user_id=user_id, is_admin=is_admin,
|
||||
)
|
||||
except ConversationNotFoundError:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Conversation not found")
|
||||
|
||||
store = svc._store # noqa: SLF001 — router needs raw event queries for streaming
|
||||
|
||||
async def event_generator():
|
||||
# ── P0-1: Open subscription BEFORE replay ──
|
||||
sub = hub.open_subscription(conversation_id)
|
||||
last_seq = after_seq
|
||||
try:
|
||||
# ── Phase 1: Replay from DB ──
|
||||
try:
|
||||
replay_events = await store.conversation_events.list_events(
|
||||
conversation_id, after_seq=last_seq,
|
||||
@@ -254,7 +256,6 @@ async def stream_conversation(
|
||||
return
|
||||
yield _format_sse_event(evt)
|
||||
last_seq = max(last_seq, int(evt.get("seq", last_seq)))
|
||||
# P0-2: stop if replay already hit a terminal event.
|
||||
if _is_terminal(evt):
|
||||
return
|
||||
except Exception:
|
||||
@@ -269,7 +270,6 @@ async def stream_conversation(
|
||||
})
|
||||
return
|
||||
|
||||
# ── P0-1: Stabilise — catch events published during replay ──
|
||||
while sub.has_pending:
|
||||
sub.mark_seen()
|
||||
late_events = await store.conversation_events.list_events(
|
||||
@@ -285,7 +285,6 @@ async def stream_conversation(
|
||||
if _is_terminal(evt):
|
||||
return
|
||||
|
||||
# ── Send snapshot marker ──
|
||||
snapshot = {
|
||||
"seq": last_seq,
|
||||
"event_id": f"snapshot_{conversation_id}_{last_seq}",
|
||||
@@ -297,7 +296,6 @@ async def stream_conversation(
|
||||
if not await request.is_disconnected():
|
||||
yield _format_sse_event(snapshot)
|
||||
|
||||
# ── Phase 2: Real-time subscription ──
|
||||
heartbeat_interval = hub.heartbeat_interval
|
||||
while True:
|
||||
if hub.is_closed:
|
||||
@@ -307,13 +305,11 @@ async def stream_conversation(
|
||||
|
||||
changed = await sub.wait_for_change(timeout=heartbeat_interval)
|
||||
if not changed:
|
||||
# Timeout → heartbeat (or hub closed).
|
||||
if hub.is_closed:
|
||||
return
|
||||
yield _format_heartbeat()
|
||||
continue
|
||||
|
||||
# Generation advanced — fetch new events from DB.
|
||||
new_events = await store.conversation_events.list_events(
|
||||
conversation_id, after_seq=last_seq,
|
||||
)
|
||||
@@ -322,7 +318,6 @@ async def stream_conversation(
|
||||
return
|
||||
yield _format_sse_event(evt)
|
||||
last_seq = max(last_seq, int(evt.get("seq", last_seq)))
|
||||
# P0-2: terminal event → close stream after sending it.
|
||||
if _is_terminal(evt):
|
||||
return
|
||||
except asyncio.CancelledError:
|
||||
@@ -334,8 +329,63 @@ async def stream_conversation(
|
||||
finally:
|
||||
sub.close()
|
||||
|
||||
return _gen()
|
||||
|
||||
|
||||
async def _get_active_run_id(store: Any, conversation_id: str) -> str | None:
|
||||
from sqlalchemy import select
|
||||
|
||||
from deerflow.persistence.maf_roundtable.model import MafRoundtableConversationRow
|
||||
async with store.session_factory() as db:
|
||||
row = (
|
||||
await db.execute(
|
||||
select(MafRoundtableConversationRow.active_run_id).where(
|
||||
MafRoundtableConversationRow.id == conversation_id,
|
||||
)
|
||||
)
|
||||
).first()
|
||||
return row[0] if row and row[0] else None
|
||||
|
||||
|
||||
@router.get("/conversations/{conversation_id}/stream")
|
||||
async def stream_conversation(
|
||||
conversation_id: str,
|
||||
request: Request,
|
||||
after_seq: int = Query(default=0, ge=0),
|
||||
):
|
||||
"""SSE stream for one conversation (§4.2)."""
|
||||
svc = _get_conversation_service(request)
|
||||
hub = _get_event_hub(request)
|
||||
user_id, is_admin = _user_ctx(request)
|
||||
|
||||
if hub is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail="SSE event hub not configured",
|
||||
)
|
||||
|
||||
stream_ready = getattr(request.app.state, "maf_conversation_stream_ready", True)
|
||||
if not stream_ready:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail=getattr(request.app.state, "maf_conversation_stream_status", "stream_not_ready"),
|
||||
)
|
||||
|
||||
try:
|
||||
await svc.get_conversation(
|
||||
conversation_id=conversation_id, user_id=user_id, is_admin=is_admin,
|
||||
)
|
||||
except ConversationNotFoundError:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Conversation not found")
|
||||
|
||||
return StreamingResponse(
|
||||
event_generator(),
|
||||
build_conversation_event_generator(
|
||||
service=svc,
|
||||
hub=hub,
|
||||
request=request,
|
||||
conversation_id=conversation_id,
|
||||
after_seq=after_seq,
|
||||
),
|
||||
media_type="text/event-stream",
|
||||
headers=_SSE_HEADERS,
|
||||
)
|
||||
@@ -119,9 +119,20 @@ async def get_health(request: Request) -> HealthResponse:
|
||||
cfg = app_config.maf_roundtable
|
||||
svc = _get_service(request)
|
||||
store_available = svc is not None and svc.store is not None
|
||||
from app.maf_roundtable.dispatch_contracts import MafDispatchCatalog
|
||||
|
||||
dispatch_ready, dispatch_status = MafDispatchCatalog(cfg).readiness()
|
||||
readiness = get_maf_write_guard_readiness(app_config=app_config)
|
||||
stream_ready = bool(getattr(request.app.state, "maf_conversation_stream_ready", False))
|
||||
stream_status = str(getattr(request.app.state, "maf_conversation_stream_status", "unknown"))
|
||||
conv_backend = str(getattr(request.app.state, "maf_conversation_event_backend", "memory"))
|
||||
return HealthResponse(
|
||||
enabled=bool(cfg.enabled and svc is not None and svc.enabled),
|
||||
enabled=bool(cfg.enabled and dispatch_ready and svc is not None and svc.enabled),
|
||||
conversation_dispatch_ready=bool(cfg.enabled and dispatch_ready and svc is not None and svc.enabled),
|
||||
conversation_dispatch_status=dispatch_status,
|
||||
conversation_stream_ready=stream_ready,
|
||||
conversation_stream_status=stream_status,
|
||||
conversation_event_backend=conv_backend,
|
||||
store_available=store_available,
|
||||
default_mode=cfg.default_mode,
|
||||
default_write_policy=cfg.default_write_policy,
|
||||
|
||||
Reference in new issue
Block a user