代码提交

This commit is contained in:
董宏凯 committed 2026-08-03 19:27:54 +08:00
1 parent d93cc62806
commit ddf83f7728
25 files changed
+4086 -424

No files matched your search

@@ -12,7 +12,7 @@ import asyncio
import logging
from collections.abc import AsyncGenerator, Callable
from contextlib import AsyncExitStack, asynccontextmanager
from typing import TYPE_CHECKING, TypeVar, cast
from typing import TYPE_CHECKING, Any, TypeVar, cast
from fastapi import FastAPI, HTTPException, Request
from langgraph.types import Checkpointer
@@ -39,6 +39,67 @@ if TYPE_CHECKING:
T = TypeVar("T")
def get_gateway_worker_topology() -> dict[str, Any]:
"""Detect the current worker count and source.
Returns ``{"worker_count": int|None, "source": str, "is_multi_worker": bool}``.
Sources checked in order: ``WEB_CONCURRENCY`` env var, ``/proc`` scan (Linux).
"""
import os as _os
wc = _os.environ.get("WEB_CONCURRENCY", "").strip()
if wc and wc.isdigit():
count = int(wc)
return {"worker_count": count, "source": "WEB_CONCURRENCY", "is_multi_worker": count > 1}
if not _os.name == "posix":
return {"worker_count": None, "source": "unknown", "is_multi_worker": False}
try:
from pathlib import Path as _Path
peers = 0
proc = _Path("/proc")
if proc.exists():
for entry in proc.iterdir():
if not entry.name.isdigit():
continue
try:
cmdline = (entry / "cmdline").read_bytes().replace(b"\x00", b" ").decode("utf-8", "ignore")
except OSError:
continue
if "uvicorn" in cmdline and "app.gateway.app" in cmdline:
peers += 1
if peers > 0:
return {"worker_count": peers, "source": "process_scan", "is_multi_worker": peers > 1}
except Exception: # noqa: BLE001
pass
return {"worker_count": None, "source": "unknown", "is_multi_worker": False}
async def _background_reconcile_loop(service: Any, *, interval: int) -> None:
"""Periodically reconcile active conversation runs."""
while True:
try:
await asyncio.sleep(interval)
await service.reconcile_active_conversations()
except asyncio.CancelledError:
raise
except Exception: # noqa: BLE001
logger.warning("[maf-roundtable] background reconcile failed", exc_info=True)
async def _startup_reconcile(service: Any) -> None:
"""One-shot reconcile of all active runs at gateway startup."""
try:
written = await service.reconcile_active_conversations()
if written:
logger.info("[maf-roundtable] startup reconcile wrote %d conversation events", written)
except Exception: # noqa: BLE001
logger.warning("[maf-roundtable] startup reconcile failed", exc_info=True)
def resolve_roundtable_gateway_mode(raw: str | None) -> str:
"""解析 ``ROUNDTABLE_JOB_GATEWAY``:**默认 ``inprocess``(真模型后台挂起)**。
@@ -321,11 +382,24 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
app.state.maf_roundtable_store = make_maf_roundtable_store(sf)
maf_enabled = False
maf_dispatch_ready = False
maf_dispatch_status = "configuration_unavailable"
try:
from app.maf_roundtable import MafRoundtableService
maf_cfg = getattr(config, "maf_roundtable", None)
maf_enabled = bool(getattr(maf_cfg, "enabled", False)) if maf_cfg is not None else False
maf_cfg = getattr(config, "maf_roundtable", None)
maf_enabled = bool(getattr(maf_cfg, "enabled", False)) if maf_cfg is not None else False
if maf_cfg is not None:
from app.maf_roundtable.dispatch_contracts import MafDispatchCatalog
maf_dispatch_ready, maf_dispatch_status = MafDispatchCatalog(maf_cfg).readiness()
if maf_enabled and not maf_dispatch_ready:
logger.error(
"[maf-roundtable] enabled but no trusted manager route is configured; "
"forcing the conversation entry OFF (%s)",
maf_dispatch_status,
)
maf_enabled = False
maf_admin_diag = (
bool(getattr(maf_cfg, "admin_diagnostic_access", True)) if maf_cfg is not None else True
)
@@ -348,7 +422,8 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
"forcing the feature OFF: %s",
cfg_exc,
)
maf_enabled = False
maf_enabled = False
maf_dispatch_ready = False
if app.state.maf_roundtable_store is not None:
app.state.maf_roundtable_service = MafRoundtableService(
app.state.maf_roundtable_store,
@@ -371,24 +446,48 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
app.state.maf_roundtable_service = None
# ── MAF roundtable V2 Conversation service + SSE event hub ──
# P0-2: multi-worker guard — memory hub cannot broadcast across
# workers. When detected, disable the conversation SSE write path
# (reads stay open for admin history review).
_topology = get_gateway_worker_topology()
_maf_cfg = getattr(config, "maf_roundtable", None)
_conv_backend = getattr(_maf_cfg, "conversation_event_backend", "memory")
_require_single = getattr(_maf_cfg, "require_single_worker_for_conversation_sse", True)
_stream_blocked = (
_conv_backend == "memory"
and _topology["is_multi_worker"]
and _require_single
)
if _stream_blocked:
logger.error(
"[maf-roundtable] conversation_event_backend=memory with %d workers "
"(%s) — SSE stream disabled. Set WEB_CONCURRENCY=1 or deploy Redis "
"Pub/Sub before enabling multi-worker conversation SSE.",
_topology["worker_count"],
_topology["source"],
)
try:
from app.maf_roundtable.conversation_event_hub import ConversationEventHub
app.state.maf_conversation_event_hub = ConversationEventHub()
if _stream_blocked:
app.state.maf_conversation_event_hub = None
else:
app.state.maf_conversation_event_hub = ConversationEventHub()
if app.state.maf_roundtable_store is not None:
from app.maf_roundtable.conversation_service import MafConversationService
from app.maf_roundtable.dispatcher import get_dispatcher
app.state.maf_conversation_service = MafConversationService(
app.state.maf_roundtable_store,
maf_config=getattr(config, "maf_roundtable", None),
event_hub=app.state.maf_conversation_event_hub,
agent_store=getattr(app.state, "agent_store", None),
app_config=config,
dispatcher=get_dispatcher(app),
enabled=maf_enabled,
)
if app.state.maf_roundtable_store is not None:
from app.maf_roundtable.conversation_service import MafConversationService
from app.maf_roundtable.dispatcher import get_dispatcher
app.state.maf_conversation_service = MafConversationService(
app.state.maf_roundtable_store,
maf_config=_maf_cfg,
event_hub=app.state.maf_conversation_event_hub,
agent_store=getattr(app.state, "agent_store", None),
app_config=config,
dispatcher=get_dispatcher(app),
enabled=maf_enabled,
)
else:
app.state.maf_conversation_service = None
except Exception: # noqa: BLE001
@@ -396,6 +495,36 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
app.state.maf_conversation_service = None
app.state.maf_conversation_event_hub = None
app.state.maf_conversation_dispatch_ready = bool(maf_enabled and maf_dispatch_ready)
app.state.maf_conversation_dispatch_status = maf_dispatch_status
app.state.maf_conversation_stream_ready = not _stream_blocked and app.state.maf_conversation_event_hub is not None
app.state.maf_conversation_stream_status = (
"multi_worker_memory_hub_unsupported" if _stream_blocked
else "ok" if app.state.maf_conversation_event_hub is not None
else "hub_not_initialized"
)
app.state.maf_conversation_event_backend = _conv_backend
# ── Background reconcile task for conversation event projection ──
_reconcile_task: asyncio.Task | None = None
if (
app.state.maf_conversation_service is not None
and maf_enabled
and not _stream_blocked
):
_reconcile_interval = getattr(_maf_cfg, "background_reconcile_interval_seconds", 10)
_reconcile_task = asyncio.create_task(
_background_reconcile_loop(
app.state.maf_conversation_service,
interval=max(5, int(_reconcile_interval)),
)
)
app.state.maf_conversation_reconcile_task = _reconcile_task
logger.info("[maf-roundtable] background reconcile started (interval=%ss)", _reconcile_interval)
# Startup reconcile: scan active/recent runs once
asyncio.create_task(_startup_reconcile(app.state.maf_conversation_service))
from deerflow.persistence.roundtable_jobs import (
make_roundtable_job_store,
)
@@ -485,14 +614,22 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
try:
yield
finally:
# P1: shut down the SSE event hub before closing the DB engine
# so active SSE streams are released cleanly.
reconcile_task = getattr(app.state, "maf_conversation_reconcile_task", None)
if reconcile_task is not None:
reconcile_task.cancel()
try:
await reconcile_task
except asyncio.CancelledError:
pass
except Exception: # noqa: BLE001
logger.warning("[maf-roundtable] reconcile task shutdown error", exc_info=True)
hub = getattr(app.state, "maf_conversation_event_hub", None)
if hub is not None:
try:
hub.shutdown()
await hub.aclose()
except Exception: # noqa: BLE001
logger.warning("[maf-roundtable] hub shutdown failed", exc_info=True)
logger.warning("[maf-roundtable] hub aclose failed", exc_info=True)
from deerflow.persistence.notifications import set_notification_store