feat(maf): Phase 5 - action coordinator, approval flow, frontend MAF roundtable pages

- Backend: action_coordinator, capability_snapshot, event_normalizer, tool_executor updates
- Backend: approved_tool_executor with write guard, action leases, migration script
- Frontend: maf-roundtable module (API client, event timeline, approval cards, fact flow panel)
- Frontend: MafRoutes, MafSessionsPage, MafRunDetailPage
- Tests: action_leases, migration_phase5, phase5_entry_e2e, real_write_e2e/http_e2e
- Docs: update implementation plan

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
董宏凯andCursor committed 2026-08-02 13:07:21 +08:00
1 parent 21c4c898e7
commit f82363d81f
55 files changed
+6304 -167

No files matched your search

@@ -6,8 +6,9 @@ import sys
import threading
from collections.abc import AsyncGenerator
from contextlib import asynccontextmanager
from datetime import datetime
from time import perf_counter
from datetime import datetime
from time import perf_counter
from typing import Any
from dotenv import load_dotenv
from fastapi import FastAPI
@@ -958,8 +959,63 @@ def _warn_if_multi_worker() -> None:
pass
@asynccontextmanager
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
async def _scan_maf_approval_expiry(app: Any) -> list[str]:
"""Expire due MAF approvals once, then resume affected workflows.
The store owns the conditional state transition and critical audit event.
Only the worker that wins that CAS receives an affected run, so repeated
scans cannot resume the same expiry twice.
"""
service = getattr(app.state, "maf_roundtable_service", None)
if service is None or not getattr(service, "enabled", False):
return []
expired = await service.store.expire_due_approvals()
run_ids = list(dict.fromkeys(
str(item["run_id"])
for item in expired
if item.get("run_id")
))
if not run_ids:
return []
from app.maf_roundtable.dispatcher import get_dispatcher
dispatcher = get_dispatcher(app)
resumed: list[str] = []
for run_id in run_ids:
run = await service.store.runs.get_run(run_id)
if run is None:
continue
session = await service.store.sessions.get_session(run["session_id"], user_id=None)
if session is None:
continue
# Dispatcher.resume is a no-op for a terminal/non-waiting run; keeping
# this call centralized preserves the restart and run-level CAS path.
await dispatcher.resume(run_id, user_id=str(session.get("user_id") or "default"))
resumed.append(run_id)
return resumed
async def _maf_approval_scanner_loop(app: Any) -> None:
"""Continuously fail closed expired MAF approvals until shutdown."""
service = getattr(app.state, "maf_roundtable_service", None)
config = getattr(service, "config", None)
interval = max(1, int(getattr(config, "approval_scan_interval_seconds", 10) or 10))
while True:
try:
await _scan_maf_approval_expiry(app)
except asyncio.CancelledError:
raise
except Exception:
logger.exception("MAF approval expiry scan failed (non-fatal)")
await asyncio.sleep(interval)
@asynccontextmanager
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
"""Application lifespan handler."""
# ?????? worker ?? ?? ?? AI ?? / ? agent ???????
@@ -1123,12 +1179,27 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
# Concurrency (system-pressure) sampler — lightweight, singleton-locked,
# runtime-toggleable via system_settings.concurrency_monitor.enabled.
try:
app.state.concurrency_sampler_task = asyncio.create_task(_concurrency_sampler_loop(app))
except Exception:
logger.exception("Concurrency sampler failed to start (non-fatal)")
# Start IM channel service if any channels are configured
try:
app.state.concurrency_sampler_task = asyncio.create_task(_concurrency_sampler_loop(app))
except Exception:
logger.exception("Concurrency sampler failed to start (non-fatal)")
# MAF-only approval expiry scanner. Legacy DeerFlow, old roundtable
# and AI-writing paths never enter this loop.
try:
maf_service = getattr(app.state, "maf_roundtable_service", None)
if maf_service is not None and getattr(maf_service, "enabled", False):
app.state.maf_approval_scanner_task = asyncio.create_task(
_maf_approval_scanner_loop(app)
)
logger.info(
"MAF approval expiry scanner started (interval=%ss)",
getattr(maf_service.config, "approval_scan_interval_seconds", 10),
)
except Exception:
logger.exception("MAF approval expiry scanner failed to start (non-fatal)")
# Start IM channel service if any channels are configured
try:
from app.channels.service import start_channel_service
@@ -1187,16 +1258,26 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
logger.exception("Leaderboard snapshot scheduler shutdown error")
concurrency_sampler_task = getattr(app.state, "concurrency_sampler_task", None)
if concurrency_sampler_task is not None:
concurrency_sampler_task.cancel()
try:
await concurrency_sampler_task
except asyncio.CancelledError:
pass
except Exception:
logger.exception("Concurrency sampler shutdown error")
# ?? 7 ????AI ?? graph ???? checkpointer / in-flight task?
if concurrency_sampler_task is not None:
concurrency_sampler_task.cancel()
try:
await concurrency_sampler_task
except asyncio.CancelledError:
pass
except Exception:
logger.exception("Concurrency sampler shutdown error")
maf_approval_scanner_task = getattr(app.state, "maf_approval_scanner_task", None)
if maf_approval_scanner_task is not None:
maf_approval_scanner_task.cancel()
try:
await maf_approval_scanner_task
except asyncio.CancelledError:
pass
except Exception:
logger.exception("MAF approval expiry scanner shutdown error")
# ?? 7 ????AI ?? graph ???? checkpointer / in-flight task?
# thread ??? LangGraph Server ? checkpointer ???lifespan ???
# ``langgraph_runtime()`` ? contextmanager ???????????????
@@ -112,18 +112,25 @@ async def get_health(request: Request) -> HealthResponse:
"""Feature-flag probe. Open read — the frontend reads ``enabled`` to
decide whether to show the MAF entry. Does not require the store.
"""
from app.maf_roundtable.service import get_maf_write_guard_readiness
from deerflow.config import get_app_config
cfg = get_app_config().maf_roundtable
app_config = get_app_config()
cfg = app_config.maf_roundtable
svc = _get_service(request)
store_available = svc is not None and svc.store is not None
readiness = get_maf_write_guard_readiness(app_config=app_config)
return HealthResponse(
enabled=cfg.enabled,
enabled=bool(cfg.enabled and svc is not None and svc.enabled),
store_available=store_available,
default_mode=cfg.default_mode,
default_write_policy=cfg.default_write_policy,
default_max_rounds=cfg.default_max_rounds,
admin_diagnostic_access=cfg.admin_diagnostic_access,
write_guard_enabled=readiness["enabled"],
write_guard_ready=readiness["ready"],
action_secret_configured=readiness["secret_configured"],
write_guard_status=readiness["status"],
)
@@ -359,6 +359,50 @@ _CONTEXT_CONFIGURABLE_KEYS: frozenset[str] = frozenset(
}
)
# These fields activate or authorize MAF WriteGuard behavior. They may only
# cross the Gateway boundary on the typed, server-created request from
# ``DeerFlowRuntimeClient``. In particular, accepting them from a public
# ``body.config`` would let a caller forge an active MAF run or a post-approval
# passthrough.
_MAF_PRIVATE_CONTEXT_KEYS: frozenset[str] = frozenset(
{
"maf_run_id",
"maf_session_id",
"maf_user_id",
"turn_index",
"write_policy",
"capability_snapshot",
"maf_approved_tool",
"maf_approved_action_id",
}
)
def _is_internal_maf_request(request: Any) -> bool:
state = getattr(request, "state", None)
return bool(getattr(state, "maf_internal", False))
def _merge_internal_maf_context(config: dict[str, Any], context: Mapping[str, Any], request: Any) -> None:
"""Forward server-generated MAF context, never public request fields."""
if not _is_internal_maf_request(request):
for container_name in ("context", "configurable"):
container = config.get(container_name)
if isinstance(container, dict):
for key in _MAF_PRIVATE_CONTEXT_KEYS:
container.pop(key, None)
return
configurable = config.setdefault("configurable", {})
runtime_context = config.setdefault("context", {})
for key in _MAF_PRIVATE_CONTEXT_KEYS:
if key not in context:
continue
if isinstance(configurable, dict):
configurable.setdefault(key, context[key])
if isinstance(runtime_context, dict):
runtime_context.setdefault(key, context[key])
def merge_run_context_overrides(config: dict[str, Any], context: Mapping[str, Any] | None) -> None:
"""Merge whitelisted keys from ``body.context`` into both ``config['configurable']``
@@ -1022,6 +1066,7 @@ async def start_run(
if value is not None and top_level_key not in body_context:
body_context[top_level_key] = value
merge_run_context_overrides(config, body_context or None)
_merge_internal_maf_context(config, body_context, request)
stream_modes = normalize_stream_modes(body.stream_mode)
@@ -54,8 +54,12 @@ from __future__ import annotations
import base64
import hashlib
import hmac
import json
import logging
import os
from datetime import UTC, datetime, timedelta
from math import isfinite
from typing import Any
from app.maf_roundtable.sanitize import sanitize, sanitize_payload
@@ -140,6 +144,90 @@ def tool_needs_approval(
return risk_val >= 3
def derive_target_key(
tool_name: str,
raw_args: Any,
capability_snapshot: dict[str, Any] | None,
agent_id: str | None = None,
) -> str:
"""Return a service-derived, opaque target lease key for a write action.
The model and browser never get a target-key input. Only the frozen,
server-built capability metadata chooses the parameter fields used to
identify a business object. Only stable scalar values are accepted;
structured or otherwise unstable values conservatively collapse to the
unidentified target. The canonical value is protected with a
domain-separated HMAC keyed by ``MAF_ACTION_SECRET_KEY`` before it is
persisted, so a database reader cannot brute-force ordinary object IDs
with a public hash.
An unclassified/missing identifier deliberately shares one conservative
key across writes (rather than guessing that different requests are safe
to parallelize). Such tools are already write/R3 by default and require
normal approval under the default policy.
"""
metadata: dict[str, Any] | None = None
snapshot_tools = (capability_snapshot or {}).get("tools")
if isinstance(snapshot_tools, list):
metadata = next(
(item for item in snapshot_tools if isinstance(item, dict) and item.get("name") == tool_name),
None,
)
if metadata is None and agent_id:
for agent in (capability_snapshot or {}).get("agents") or []:
if not isinstance(agent, dict) or agent.get("agent_id") != agent_id:
continue
for item in agent.get("tools") or []:
if isinstance(item, dict) and item.get("name") == tool_name:
metadata = item
break
if metadata is not None:
break
fields = metadata.get("target_key_fields") if isinstance(metadata, dict) else None
namespace = metadata.get("target_key_namespace") if isinstance(metadata, dict) else None
if not isinstance(fields, list) or not fields or not isinstance(raw_args, dict):
canonical = {"scope": "unidentified"}
else:
valid_fields = [
field for field in fields
if isinstance(field, str) and field != "target_key" and field in raw_args
]
if len(valid_fields) != len(fields) or any(
not _is_stable_scalar(raw_args[field]) for field in valid_fields
):
canonical = {"scope": "unidentified"}
else:
canonical = {
"scope": "identified",
"tool": tool_name,
"namespace": namespace if isinstance(namespace, str) and namespace else "default",
"target": {field: raw_args[field] for field in sorted(valid_fields)},
}
encoded = json.dumps(canonical, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
raw_secret = os.getenv("MAF_ACTION_SECRET_KEY", "")
if not raw_secret.strip():
raise MafActionSecretKeyError(
"MAF_ACTION_SECRET_KEY is required before deriving a real-write target key"
)
domain = "maf-target-key:v1"
digest = hmac.new(
raw_secret.encode("utf-8"),
domain.encode("ascii") + b"\x00" + encoded.encode("utf-8"),
hashlib.sha256,
).hexdigest()
return f"maf-target:v1:{digest}"
def _is_stable_scalar(value: Any) -> bool:
"""Whether a configured target field can safely identify one target."""
if value is None or isinstance(value, (str, bool, int)):
return True
return isinstance(value, float) and isfinite(value)
# ── Encryption ─────────────────────────────────────────────────────────────
@@ -230,6 +318,10 @@ class ActionCoordinator:
self._store = service.store
self._user_id = user_id
def _approval_expires_at(self) -> datetime:
seconds = int(getattr(self._service.config, "approval_timeout_seconds", 3600) or 3600)
return datetime.now(UTC) + timedelta(seconds=seconds)
# ── Post-turn: capture intercepted write tool calls ──────────────────
async def intercept_write_tool_calls(
@@ -268,6 +360,7 @@ class ActionCoordinator:
_side_effect, risk_level = classify_tool(tool_name, capability_snapshot, agent_id)
idempotency_key = f"{run_id}:{agent_id}:{turn_index}:{tool_call_id or tool_name}"
target_key = derive_target_key(tool_name, raw_args, capability_snapshot, agent_id)
# Sanitize args for audit visibility (never expose raw args).
sanitized_args = sanitize(raw_args)
@@ -290,7 +383,7 @@ class ActionCoordinator:
logical_action_id=tool_call_id or None,
idempotency_key=idempotency_key,
risk_level=risk_level,
target_key=None,
target_key=target_key,
turn_index=turn_index,
arguments_sanitized=sanitized_args,
arguments_encrypted=encrypted_args,
@@ -303,6 +396,7 @@ class ActionCoordinator:
},
audit_event_agent_id=agent_id,
audit_event_turn_index=turn_index,
approval_expires_at=self._approval_expires_at(),
)
created.append(action)
@@ -322,7 +416,10 @@ class ActionCoordinator:
approved. This guarantees at-most-once execution under concurrent
``resume()`` / retries / restart.
"""
return await self._store.actions.claim_for_execution(action_id)
lease_seconds = int(getattr(self._service.config, "target_lease_seconds", 900) or 900)
return await self._store.claim_action_for_execution(
action_id, lease_seconds=lease_seconds
)
async def record_result(
self,
@@ -344,7 +441,7 @@ class ActionCoordinator:
is a no-op — it must NOT overwrite the real result.
"""
sanitized_result = sanitize_payload(result) if result else None
await self._store.actions.record_terminal_result(
await self._store.record_action_terminal_result(
action_id,
status=status,
result=sanitized_result,
@@ -66,7 +66,7 @@ def _seat_tools_for_agent(
session_model_name: str | None,
session_reasoning_effort: str | None,
requested_excluded: list[str],
) -> tuple[list[str], list[str], list[dict[str, str]]]:
) -> tuple[list[str], list[str], list[dict[str, Any]]]:
"""Resolve allowed names, effective excluded, and tool metadata for one seat."""
agent_config = load_agent_config(agent_id)
tool_groups = getattr(agent_config, "tool_groups", None) if agent_config else None
@@ -88,10 +88,19 @@ def _seat_tools_for_agent(
effective_excluded = sorted(excluded_set & allowed_set)
effective_allowed = sorted(allowed_set - set(effective_excluded))
tools_meta: list[dict[str, str]] = []
tools_meta: list[dict[str, Any]] = []
maf_tool_metadata = getattr(
getattr(app_config, "maf_roundtable", None), "tool_metadata", None
) or {}
for name in effective_allowed:
extra = _config_extra_for_tool(app_config, name)
tools_meta.append(resolve_tool_metadata(name, config_extra=extra))
tools_meta.append(
resolve_tool_metadata(
name,
config_extra=extra,
maf_metadata=maf_tool_metadata,
)
)
return effective_allowed, effective_excluded, tools_meta
@@ -108,7 +117,7 @@ async def build_trusted_capability_snapshot(
) -> dict[str, Any]:
"""Build the frozen snapshot stored on the session and used by WriteGuard."""
agents_out: list[dict[str, Any]] = []
flat_tools: dict[str, dict[str, str]] = {}
flat_tools: dict[str, dict[str, Any]] = {}
for p in participants:
agent_id = str(p.get("agent_id") or "")
@@ -137,6 +137,16 @@ def make_action_rejected(agent_id: str, turn_index: int, *, action_id: str, appr
return _make("action.rejected", agent_id, turn_index, {"action_id": action_id, "approval_id": approval_id, "decided_by": decided_by})
def make_action_unknown(agent_id: str, turn_index: int, *, action_id: str, reason_code: str) -> dict[str, Any]:
"""Record an inconclusive external outcome without leaking tool arguments."""
return _make("action.unknown", agent_id, turn_index, {"action_id": action_id, "reason_code": reason_code})
def make_action_waiting_target_lease(agent_id: str, turn_index: int, *, action_id: str, tool_name: str) -> dict[str, Any]:
"""A second approved write is serialized behind an active target lease."""
return _make("action.waiting_target_lease", agent_id, turn_index, {"action_id": action_id, "tool_name": tool_name})
# ── workflow checkpoint events ─────────────────────────────────────────────
@@ -78,6 +78,10 @@ class _InternalState:
def __init__(self, principal: ExecutionPrincipal) -> None:
self.user = _PrincipalUser(principal)
self.auth = None
# This marker is set only on the server-created request used by the
# MAF dispatcher. Gateway code uses it to distinguish trusted MAF
# execution context from user-supplied run configuration.
self.maf_internal = True
@dataclass
@@ -193,8 +193,6 @@ class ActionResponse(BaseModel):
agent_id: str | None = None
tool_name: str
logical_action_id: str | None = None
idempotency_key: str
target_key: str | None = None
risk_level: str
status: str
arguments: dict[str, Any] | None = None
@@ -255,3 +253,7 @@ class HealthResponse(BaseModel):
default_write_policy: str
default_max_rounds: int
admin_diagnostic_access: bool
write_guard_enabled: bool
write_guard_ready: bool
action_secret_configured: bool
write_guard_status: str
@@ -79,6 +79,56 @@ class MafConfigurationError(RuntimeError):
"""
def get_maf_write_guard_readiness(*, app_config: Any | None = None) -> dict[str, Any]:
"""Return non-sensitive WriteGuard readiness information for MAF health.
A missing secret intentionally does not disable read-only MAF work:
WriteGuard fails closed before any external write can execute. ``ready``
is therefore the signal that the complete approval + encryption path is
usable, while ``enabled`` describes whether the middleware is mounted.
"""
import os
from deerflow.config.write_guard_config import get_write_guard_config
from deerflow.reflection import resolve_variable
wg = getattr(app_config, "write_guard", None) if app_config is not None else None
if wg is None:
wg = get_write_guard_config()
provider = getattr(wg, "provider", None)
provider_configured = bool(
provider
and getattr(provider, "policy_use", "")
and getattr(provider, "interceptor_use", "")
)
bridge_resolvable = False
if provider_configured:
try:
resolve_variable(provider.policy_use)
resolve_variable(provider.interceptor_use)
bridge_resolvable = True
except Exception: # noqa: BLE001 - health must remain safe and non-fatal
bridge_resolvable = False
secret_configured = bool((os.getenv("MAF_ACTION_SECRET_KEY") or "").strip())
if not getattr(wg, "enabled", False):
status = "disabled"
elif not provider_configured or not bridge_resolvable:
status = "provider_unavailable"
elif not secret_configured:
status = "missing_secret"
else:
status = "ready"
return {
"enabled": bool(getattr(wg, "enabled", False)),
"ready": status == "ready",
"secret_configured": secret_configured,
"status": status,
}
def validate_maf_prerequisites() -> None:
"""Fail loudly if MAF is enabled without its full safety stack in place.
@@ -86,14 +136,13 @@ def validate_maf_prerequisites() -> None:
1. ``write_guard.enabled`` is true (the approval guard is mounted);
2. ``write_guard.provider`` is complete (``policy_use`` + ``interceptor_use``);
3. the bridge factories actually resolve (the app-layer guard is importable);
4. ``MAF_ACTION_SECRET_KEY`` is set (action params are encrypted with it).
4. secret readiness is reported separately; a missing secret must not
disable read-only MAF work, while writes fail closed before execution.
Raises ``MafConfigurationError`` on the first unmet condition. Callers on
the startup path treat that as "force the feature off"; direct unit-test
construction of :class:`MafRoundtableService` does NOT call this.
"""
import os
from deerflow.config.write_guard_config import get_write_guard_config
from deerflow.reflection import resolve_variable
@@ -116,10 +165,6 @@ def validate_maf_prerequisites() -> None:
raise MafConfigurationError(
f"write_guard bridge factories are not resolvable: {exc}"
) from exc
if not (os.getenv("MAF_ACTION_SECRET_KEY") or "").strip():
raise MafConfigurationError(
"maf_roundtable.enabled=true requires the MAF_ACTION_SECRET_KEY env var"
)
# Session statuses that may be edited (config changes).
@@ -150,24 +195,10 @@ class MafRoundtableService:
# defaults when no config is injected (keeps unit tests lightweight).
self._config = config or MafRoundtableConfig()
# Startup validation: if the MAF feature is enabled, the
# ``MAF_ACTION_SECRET_KEY`` env var MUST be set — external-write
# action params are encrypted with it and there is no default key.
# A missing key forces the feature off (write paths then return 503)
# rather than running with a publicly-known key. This never aborts
# Gateway startup (the feature is experimental / opt-in).
if enabled:
import os
key = os.getenv("MAF_ACTION_SECRET_KEY", "")
if not key or not key.strip():
logger.error(
"[maf-roundtable] MAF_ACTION_SECRET_KEY is not set — "
"disabling the feature (write paths will return 503). "
"Set a 32+ char random string before enabling "
"maf_roundtable.enabled."
)
enabled = False
# A missing MAF_ACTION_SECRET_KEY does not disable read-only MAF
# operation. The action coordinator raises before creating or
# executing an external-write action, so writes fail closed until the
# deployment injects the secret through the environment.
self._enabled = enabled
@property
@@ -539,10 +570,17 @@ class MafRoundtableService:
async def list_actions(
self, *, run_id: str, user_id: str | None, is_admin: bool = False
) -> list[dict[str, Any]]:
from deerflow.persistence.maf_roundtable.actions import action_to_public_dict
run = await self.get_run(run_id=run_id, user_id=user_id, is_admin=is_admin)
return await self._store.actions.list_actions(run_id=run["id"])
return [
action_to_public_dict(action)
for action in await self._store.actions.list_actions(run_id=run["id"])
]
async def get_action(self, *, action_id: str, user_id: str | None, is_admin: bool = False) -> dict[str, Any]:
from deerflow.persistence.maf_roundtable.actions import action_to_public_dict
action = await self._store.actions.get_action(action_id)
if action is None:
raise MafNotFoundError("action", action_id)
@@ -550,7 +588,7 @@ class MafRoundtableService:
session = await self._store.sessions.get_session(action["session_id"], user_id=scope)
if session is None:
raise MafNotFoundError("action", action_id)
return action
return action_to_public_dict(action)
# ── approvals ──────────────────────────────────────────────────────────
@@ -558,6 +596,8 @@ class MafRoundtableService:
self, *, run_id: str, user_id: str | None, is_admin: bool = False
) -> list[dict[str, Any]]:
run = await self.get_run(run_id=run_id, user_id=user_id, is_admin=is_admin)
# Reads are observational. Expiry is performed by the dedicated
# scanner; decision/resume paths retain the fail-closed pre-CAS check.
return await self._store.approvals.list_approvals(run_id=run["id"])
async def decide_approval(
@@ -579,6 +619,11 @@ class MafRoundtableService:
if is_admin and await self._store.sessions.get_session(approval["session_id"], user_id=None) is not None:
raise MafForbiddenError("approval", approval_id)
raise MafNotFoundError("approval", approval_id)
# The timeout transition is server-owned and happens before the
# decision CAS, so an expired approval can never be revived by a late
# browser request.
if approval.get("run_id"):
await self._store.expire_due_approvals(run_id=approval["run_id"])
try:
updated = await self._store.approvals.decide(approval_id, decision=decision, decided_by=user_id)
except ValueError as exc:
@@ -113,6 +113,15 @@ class ToolExecutor:
)
return {"status": "failed", "error": f"action state machine error: {exc}"}
if claim == "target_busy":
logger.info("ToolExecutor: target lease is held for action %s", action_id)
return {"status": "waiting_target_lease", "error": "target lease is held by another action"}
if claim == "target_reconciliation_required":
logger.warning("ToolExecutor: expired target lease requires reconciliation for %s", action_id)
return {
"status": "unknown",
"error": "target lease expired during another action; reconciliation required",
}
if claim != "claimed":
existing = await self._coordinator._store.actions.get_action(action_id) # noqa: SLF001
if claim == "in_progress":
@@ -128,9 +137,8 @@ class ToolExecutor:
return {"status": claim, "action": existing}
if not approved_tool:
await self._coordinator.record_result(
action_id, status="failed",
result={"error": "no approved tool name — cannot execute"},
await self._record_result_or_log(
action_id, status="failed", result={"error": "no approved tool name — cannot execute"}
)
return {"status": "failed", "error": "no approved tool name"}
@@ -147,11 +155,14 @@ class ToolExecutor:
"ToolExecutor: action %s params unavailable (%s): %s",
action_id, exc.reason, safe_msg,
)
await self._coordinator.record_result(
persisted = await self._record_result_or_log(
action_id, status="failed",
result={"error": f"action secret unavailable: {exc.reason}"},
)
return {"status": "failed", "error": safe_msg}
return {
"status": "failed" if persisted else "unknown",
"error": safe_msg if persisted else "action result persistence failed",
}
# 3. Read the stored action record (authoritative turn_index + the
# run/session/agent/tool the harness will verify against).
@@ -191,34 +202,83 @@ class ToolExecutor:
"ToolExecutor: direct tool call '%s' raised: %s",
approved_tool, safe_msg,
)
await self._coordinator.record_result(
action_id, status="failed", result={"error": safe_msg},
# An exception can arrive after the remote request was accepted
# (network disconnect / timeout), so fail closed as ``unknown``.
persisted = await self._record_result_or_log(
action_id, status="unknown", result={"error": "tool_transport_exception"}
)
return {"status": "failed", "error": safe_msg}
return {
"status": "unknown",
"error": safe_msg if persisted else "action result persistence failed",
}
# 5. Record terminal result (state-machine constrained: only
# running → terminal; a lost race is a no-op).
outcome_status = outcome.get("status")
if outcome_status == "succeeded":
await self._coordinator.record_result(
action_id, status="succeeded",
result={"tool_result": outcome.get("result")},
tool_result = outcome.get("result")
external_object_id = (
str(tool_result.get("external_object_id"))
if isinstance(tool_result, dict)
and tool_result.get("external_object_id")
else None
)
return {"status": "succeeded", "tool_result": outcome.get("result")}
persisted = await self._record_result_or_log(
action_id, status="succeeded",
result={"tool_result": tool_result},
external_object_id=external_object_id,
)
if not persisted:
# The external side effect succeeded but its local terminal
# record did not. Never retry; the retained running lease is
# later reconciled to unknown by recovery.
return {"status": "unknown", "error": "action result persistence failed"}
return {"status": "succeeded", "tool_result": tool_result}
elif outcome_status == "unknown":
# Inconclusive external result (timeout / disconnect / gateway):
# record ``unknown`` — never auto-retry, a human must reconcile.
error = outcome.get("error", "inconclusive external result")
await self._coordinator.record_result(
persisted = await self._record_result_or_log(
action_id, status="unknown", result={"error": error},
)
return {"status": "unknown", "error": error}
return {"status": "unknown", "error": error if persisted else "action result persistence failed"}
else:
error = outcome.get("error", "tool execution failed")
await self._coordinator.record_result(
persisted = await self._record_result_or_log(
action_id, status="failed", result={"error": error},
)
return {"status": "failed", "error": error}
return {
"status": "failed" if persisted else "unknown",
"error": error if persisted else "action result persistence failed",
}
async def _record_result_or_log(
self,
action_id: str,
*,
status: str,
result: dict[str, Any],
external_object_id: str | None = None,
) -> bool:
"""Persist a terminal outcome without opening a retry path after I/O.
If this write fails after a real tool reports success, returning an
``unknown`` outcome lets the parent run stop for reconciliation while
the still-held lease prevents a replay of the external call.
"""
try:
await self._coordinator.record_result(
action_id,
status=status,
result=result,
external_object_id=external_object_id,
)
except Exception: # noqa: BLE001
logger.exception(
"ToolExecutor: could not persist terminal result for action %s", action_id
)
return False
return True
async def record_unknown(self, action_id: str, *, reason: str) -> None:
"""Record an ``unknown`` outcome (network timeout / inconclusive).
@@ -230,6 +290,6 @@ class ToolExecutor:
"ToolExecutor: action %s outcome unknown — %s (no auto-retry)",
action_id, reason,
)
await self._coordinator.record_result(
action_id, status="unknown", result={"reason": reason},
await self._record_result_or_log(
action_id, status="unknown", result={"reason": reason}
)
@@ -316,6 +316,10 @@ class WorkflowRunner:
logger.warning("Cannot resume run %s: no pending_approvals in checkpoint", run_id)
return run
# Expire due approvals before inspecting the checkpoint. This is a
# fail-closed server transition; a late HTTP approval cannot revive it.
await self._service.store.expire_due_approvals(run_id=run_id)
# Check if any approval is still pending — if so, don't resume yet.
any_pending = False
for pa in state.pending_approvals:
@@ -365,14 +369,15 @@ class WorkflowRunner:
from app.maf_roundtable.event_normalizer import make_action_rejected
# Conditional transition (waiting_approval → rejected) — never
# overwrite a terminal state via a generic update.
await self._service.store.actions.mark_rejected(action_id)
await self._adapter._emit_critical(run_id, make_action_rejected(
pa.get("agent_id", ""),
state.current_round,
action_id=action_id,
approval_id=approval_id,
decided_by=approval.get("decided_by", ""),
))
changed = await self._service.store.actions.mark_rejected(action_id)
if changed:
await self._adapter._emit_critical(run_id, make_action_rejected(
pa.get("agent_id", ""),
state.current_round,
action_id=action_id,
approval_id=approval_id,
decided_by=approval.get("decided_by", ""),
))
action_outcomes.append({"action_id": action_id, "status": "rejected"})
# Phase-3 P1-3: move the ledger row out of pending into the
# failed/unknown partition as rejected.
@@ -478,6 +483,37 @@ class WorkflowRunner:
# P0-3: determine parent run status from action outcomes.
statuses = {o.get("status") for o in action_outcomes}
# A target held by another approved write remains pending for an
# explicit later resume. It is never retried automatically.
if "waiting_target_lease" in statuses:
from app.maf_roundtable.event_normalizer import make_action_waiting_target_lease
state.status = "waiting_approval"
for outcome in action_outcomes:
if outcome.get("status") == "waiting_target_lease":
await self._adapter._emit_critical(
run_id,
make_action_waiting_target_lease(
next(
(str(pa.get("agent_id") or "") for pa in state.pending_approvals if pa.get("action_id") == outcome.get("action_id")),
"",
),
state.current_round,
action_id=str(outcome.get("action_id") or ""),
tool_name=str(outcome.get("tool_name") or ""),
),
)
await self._service.store.runs.update_run(run_id, {
"status": "waiting_approval",
"maf_checkpoint": state.to_dict(),
})
await self._service.store.sessions.update_session(
session_id,
user_id=self._user_id,
fields={"status": "waiting_approval", "fact_ledger": state.fact_ledger},
)
return await self._service.store.runs.get_run(run_id)
# While any action is still in_progress (another executor running it),
# the parent run stays running — never write finished_at, never finalize.
if "in_progress" in statuses:
@@ -572,6 +608,44 @@ class WorkflowRunner:
if session is None:
return run
expired_actions = await self._service.store.recover_expired_action_leases(run_id=run_id)
if expired_actions:
from app.maf_roundtable.event_normalizer import make_action_unknown
expired_ids = {action["id"] for action in expired_actions}
for action in expired_actions:
record_action_transition(
state.fact_ledger,
action_id=action["id"],
tool_name=action.get("tool_name", ""),
status="unknown",
agent_id=action.get("agent_id", ""),
)
await self._adapter._emit_critical(
run_id,
make_action_unknown(
action.get("agent_id", ""),
int(action.get("turn_index") or state.current_round),
action_id=action["id"],
reason_code="target_lease_expired_during_execution",
),
)
state.pending_approvals = [
item for item in state.pending_approvals
if item.get("action_id") not in expired_ids
]
state.status = "waiting_input"
await self._service.store.runs.update_run(run_id, {
"status": "waiting_input",
"maf_checkpoint": state.to_dict(),
})
await self._service.store.sessions.update_session(
state.session_id,
user_id=self._user_id,
fields={"status": "waiting_input", "fact_ledger": state.fact_ledger},
)
return await self._service.store.runs.get_run(run_id)
# Ensure threads still bound.
participants = await self._service.store.participants.list_participants(state.session_id)
thread_map = await self._adapter.ensure_threads(state.session_id, run_id, participants)
+15 -2
View File
@@ -558,13 +558,23 @@ task_deeplink:
# - 前端入口隐藏
# 详见 frontend-web/docs/microsoft-agent-framework-多智能体会商-实施计划.md
maf_roundtable:
enabled: false # 部署级特性开关;首期仅管理员/白名单可见
enabled: true # 启用独立 MAF 会商页面;不影响旧 roundtable 功能
default_mode: structured # structured(固定流程)| group_chat(动态群聊)
default_write_policy: confirm_high_risk # auto | confirm_high_risk | confirm_all_writes
default_max_rounds: 8 # 群聊模式轮次硬上限
admin_diagnostic_access: true # 管理员是否可跨用户读取所有 MAF 会商用于诊断(仅读,不含写)
seat_timeout_seconds: 600 # 结构化流程单席位超时(秒)
approval_scan_interval_seconds: 10 # 审批主动过期扫描间隔(秒)
schema_repair_enabled: true # 席位 JSON 校验失败时允许一次定向修复
# 必须使用 MCP 实际发现到的运行时 tool name;不要改成 server 名或前缀猜测。
# 当前部署仅启用检索 MCP,因此这里只登记两个只读工具。
tool_metadata:
weknora-retrieval_weknora_list_knowledge_bases:
side_effect: read
risk_level: R0
weknora-retrieval_weknora_search:
side_effect: read
risk_level: R0
# ── WriteGuard 中间件配置 ────────────────────────────────────────────────────
# 与 maf_roundtable 联动:MAF 会商运行时,WriteGuard 中间件拦截外部写工具调用,
@@ -573,7 +583,10 @@ maf_roundtable:
# policy_use / interceptor_use 指向 app.maf_roundtable 的具体实现(通过
# resolve_variable 解析),保证 harness 层不导入 app.*。
write_guard:
enabled: false
# MAF-only WriteGuard is mounted in production. Legacy runs without
# maf_run_id pass through unchanged; a missing MAF_ACTION_SECRET_KEY makes
# external writes fail closed while read-only tools and skills remain usable.
enabled: true
provider:
policy_use: app.maf_roundtable.write_guard_bridge:make_policy
interceptor_use: app.maf_roundtable.write_guard_bridge:make_interceptor
@@ -17,15 +17,74 @@
from __future__ import annotations
import re
from typing import Literal
from pydantic import BaseModel, Field
from pydantic import BaseModel, Field, field_validator
# ── shared enums (config + API schemas) ───────────────────────────────────
MafMode = Literal["structured", "group_chat"]
MafWritePolicy = Literal["auto", "confirm_high_risk", "confirm_all_writes"]
MafParticipantRole = Literal["manager", "seat"]
MafToolSideEffect = Literal["read", "skill", "write"]
MafRiskLevel = Literal["R0", "R1", "R2", "R3", "R4"]
class MafToolMetadata(BaseModel):
"""Deployment-owned side-effect classification for one actual tool name.
This lives in the MAF configuration rather than in the browser request or
model output. The key in ``MafRoundtableConfig.tool_metadata`` must be the
exact runtime tool name (including the MCP server prefix when applicable).
"""
side_effect: MafToolSideEffect = Field(
description="Tool side effect class used by the MAF WriteGuard",
)
risk_level: MafRiskLevel = Field(
description="External action risk level used by MAF approval policy",
)
target_key_fields: list[str] = Field(
default_factory=list,
description=(
"仅服务端配置的、用于派生目标租约键的顶层参数字段。"
"参数值只参与稳定哈希,绝不直接写入 target_key。"
),
)
target_key_namespace: str = Field(
default="default",
min_length=1,
max_length=96,
description="服务端目标键命名空间;用于隔离不同租户/业务域。",
)
@field_validator("target_key_fields")
@classmethod
def _validate_target_key_fields(cls, fields: list[str]) -> list[str]:
"""Only allow explicit top-level server-owned argument names.
In particular, a model/client supplied ``target_key`` is never a
valid input. Nested paths are deliberately unsupported: a production
adapter must register the exact stable business-id fields it owns.
"""
valid = re.compile(r"^[A-Za-z_][A-Za-z0-9_]{0,63}$")
normalized: list[str] = []
for field in fields:
if not isinstance(field, str) or not valid.fullmatch(field):
raise ValueError("target_key_fields must be simple top-level argument names")
if field == "target_key":
raise ValueError("target_key must not be supplied as a target key input")
if field not in normalized:
normalized.append(field)
return normalized
@field_validator("target_key_namespace")
@classmethod
def _validate_target_key_namespace(cls, namespace: str) -> str:
if not re.fullmatch(r"[A-Za-z0-9_.:-]{1,96}", namespace):
raise ValueError("target_key_namespace contains unsupported characters")
return namespace
class MafRoundtableConfig(BaseModel):
@@ -74,11 +133,39 @@ class MafRoundtableConfig(BaseModel):
default=True,
description="席位输出未通过 SeatTurnOutput 校验时是否允许一次定向格式修复",
)
# Server-trusted metadata keyed by the exact runtime tool name. Do not
# infer this from MCP server names or prefixes: the actual discovered name
# is what the capability snapshot and runtime WriteGuard see.
tool_metadata: dict[str, MafToolMetadata] = Field(
default_factory=dict,
description="MAF 专用工具副作用/风险元数据,键必须是实际运行时 tool name",
)
approval_timeout_seconds: int = Field(
default=3600,
ge=60,
le=604800,
description="外部写入审批有效期(秒);到期后自动拒绝,绝不执行。",
)
approval_scan_interval_seconds: int = Field(
default=10,
ge=1,
le=3600,
description="MAF approval expiry scan interval in seconds",
)
target_lease_seconds: int = Field(
default=900,
ge=30,
le=7200,
description="外部业务目标写入租约有效期(秒);过期运行中的动作转 unknown。",
)
__all__ = [
"MafMode",
"MafParticipantRole",
"MafRiskLevel",
"MafRoundtableConfig",
"MafToolMetadata",
"MafToolSideEffect",
"MafWritePolicy",
]
@@ -1,9 +1,9 @@
"""Microsoft Agent Framework (MAF) roundtable experiment persistence.
Six independent tables — **no foreign keys to the legacy ``roundtable_*``
Nine independent tables — **no foreign keys to the legacy ``roundtable_*``
tables** — so the existing multi-agent roundtable keeps its current behavior,
schema, and recovery semantics untouched. The MAF experiment is fully
isolated: it has its own session / participant / run / event / action /
isolated: it has its own session / participant / run / event / action / lease /
approval tables.
Boundary: this package lives in the harness layer (``deerflow.*``) and never
@@ -18,7 +18,8 @@ Tables (see the implementation plan §8):
- ``maf_roundtable_runs`` — one execution run of a session
- ``maf_roundtable_events`` — monotonic per-run event stream (``(run_id, seq)`` unique)
- ``maf_roundtable_actions`` — external write / tool actions (idempotency-keyed)
- ``maf_roundtable_approvals`` — pending human-approval requests
- ``maf_roundtable_action_leases`` — active target-write leases
- ``maf_roundtable_approvals`` — pending human-approval requests
- ``maf_roundtable_seat_runs`` — MAF↔DeerFlow run mapping (survives restart)
- ``maf_roundtable_action_secrets`` — encrypted actual execution params (server-only)
"""
@@ -32,6 +33,7 @@ from deerflow.persistence.maf_roundtable.actions import MafActionRepository
from deerflow.persistence.maf_roundtable.approvals import MafApprovalRepository
from deerflow.persistence.maf_roundtable.events import MafEventRepository
from deerflow.persistence.maf_roundtable.model import (
MafRoundtableActionLeaseRow,
MafRoundtableActionRow,
MafRoundtableActionSecretRow,
MafRoundtableApprovalRow,
@@ -54,6 +56,7 @@ __all__ = [
"MafEventRepository",
"MafParticipantRepository",
"MafRoundtableActionRow",
"MafRoundtableActionLeaseRow",
"MafRoundtableActionSecretRow",
"MafRoundtableApprovalRow",
"MafRoundtableEventRow",
@@ -10,6 +10,7 @@ store does not re-redact.
from __future__ import annotations
from collections.abc import Mapping
from datetime import UTC, datetime
from typing import Any
from uuid import uuid4
@@ -50,6 +51,19 @@ def action_to_dict(row: MafRoundtableActionRow) -> dict[str, Any]:
}
def action_to_public_dict(action: Mapping[str, Any]) -> dict[str, Any]:
"""Return the allow-listed action representation exposed by MAF HTTP APIs.
Target lease keys and idempotency keys are persistence/coordinator
details. They must never cross the public boundary.
"""
public = dict(action)
public.pop("target_key", None)
public.pop("idempotency_key", None)
return public
# Back-compat alias for internal callers that used the old private name.
_to_dict = action_to_dict
@@ -1,9 +1,9 @@
"""ORM models for the MAF (Microsoft Agent Framework) roundtable experiment.
Six independent tables — **no foreign keys to the legacy ``roundtable_*``
Nine independent tables — **no foreign keys to the legacy ``roundtable_*``
tables** (``roundtable_jobs`` / ``roundtable_drafts`` /
``roundtable_task_drafts``). The MAF experiment is fully isolated: it has its
own session / participant / run / event / action / approval tables, keyed by
own session / participant / run / event / action / lease / approval tables, keyed by
its own surrogate ids, so the legacy multi-agent roundtable keeps its current
behavior, schema, and recovery semantics untouched.
@@ -14,6 +14,7 @@ Tables (see the implementation plan §8):
- ``maf_roundtable_runs`` — one execution run of a session
- ``maf_roundtable_events`` — monotonic per-run event stream (``(run_id, seq)`` unique)
- ``maf_roundtable_actions`` — external write / tool actions (idempotency-keyed)
- ``maf_roundtable_action_leases`` — active target-write leases (one per target key)
- ``maf_roundtable_approvals`` — pending human-approval requests
User isolation: ``maf_roundtable_sessions.user_id`` is the owner; all child
@@ -263,7 +264,42 @@ class MafRoundtableActionRow(Base):
)
# ── 6. approvals ──────────────────────────────────────────────────────────
# ── 6. target leases ──────────────────────────────────────────────────────
class MafRoundtableActionLeaseRow(Base):
"""One active, short-lived lease for a server-derived external target key.
The key is opaque (a stable SHA-256 digest) and is never sourced from a
browser/model field. ``target_key`` is the primary key, which makes the
database — not an in-process mutex — arbitrate concurrent writes across
workers and restarts. A lease belongs to exactly one action and is removed
with a known terminal result; an ``unknown`` result deliberately retains a
manual-reconciliation barrier.
"""
__tablename__ = "maf_roundtable_action_leases"
__table_args__ = (
UniqueConstraint("action_id", name="uq_maf_action_leases_action"),
)
target_key: Mapped[str] = mapped_column(String(255), primary_key=True)
action_id: Mapped[str] = mapped_column(
String(64),
ForeignKey("maf_roundtable_actions.id", ondelete="CASCADE"),
nullable=False,
index=True,
)
lease_expires_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), nullable=False, index=True
)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now)
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), nullable=False, default=_now, onupdate=_now
)
# ── 7. approvals ──────────────────────────────────────────────────────────
class MafRoundtableApprovalRow(Base):
@@ -297,7 +333,7 @@ class MafRoundtableApprovalRow(Base):
)
# ── 7. seat runs (Phase 2: MAF↔DeerFlow run mapping) ─────────────────────
# ── 8. seat runs (Phase 2: MAF↔DeerFlow run mapping) ─────────────────────
class MafRoundtableSeatRunRow(Base):
@@ -370,7 +406,7 @@ class MafRoundtableSeatRunRow(Base):
)
# ── 8. action secrets (encrypted actual execution params, server-only) ────
# ── 9. action secrets (encrypted actual execution params, server-only) ────
class MafRoundtableActionSecretRow(Base):
@@ -403,6 +439,7 @@ class MafRoundtableActionSecretRow(Base):
__all__ = [
"MafRoundtableActionRow",
"MafRoundtableActionLeaseRow",
"MafRoundtableActionSecretRow",
"MafRoundtableApprovalRow",
"MafRoundtableEventRow",
@@ -14,19 +14,24 @@ another's transactions.
from __future__ import annotations
from datetime import UTC, datetime
from datetime import UTC, datetime, timedelta
from typing import Any
from uuid import uuid4
from sqlalchemy import func, select, update
from sqlalchemy import delete, func, select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.maf_roundtable.action_secrets import MafActionSecretRepository
from deerflow.persistence.maf_roundtable.actions import MafActionRepository, action_to_dict
from deerflow.persistence.maf_roundtable.actions import (
MafActionRepository,
MafActionStateMachineError,
action_to_dict,
)
from deerflow.persistence.maf_roundtable.approvals import MafApprovalRepository
from deerflow.persistence.maf_roundtable.events import MafEventRepository
from deerflow.persistence.maf_roundtable.model import (
MafRoundtableActionLeaseRow,
MafRoundtableActionRow,
MafRoundtableActionSecretRow,
MafRoundtableApprovalRow,
@@ -315,6 +320,7 @@ class MafRoundtableStore:
approval_payload: dict[str, Any] | None,
audit_event_agent_id: str,
audit_event_turn_index: int,
approval_expires_at: datetime | None = None,
) -> dict[str, Any]:
"""Create an action + secret + approval + audit event in ONE transaction.
@@ -432,6 +438,7 @@ class MafRoundtableStore:
status="pending",
summary=approval_summary,
payload_redacted_json=approval_payload,
expires_at=approval_expires_at,
created_at=now,
updated_at=now,
))
@@ -482,10 +489,308 @@ class MafRoundtableStore:
return result
raise RuntimeError(
f"create_pending_action_bundle failed after {_MAX_BUNDLE_RETRIES} attempts "
f"for idempotency_key={idempotency_key}"
f"create_pending_action_bundle failed after {_MAX_BUNDLE_RETRIES} attempts"
) from last_error
async def claim_action_for_execution(self, action_id: str, *, lease_seconds: int) -> str:
"""Atomically claim an approved action and its external-target lease.
The action CAS and the target lease live in one transaction. A second
approved action for the same target receives ``target_busy`` and is not
executed. An expired lease owned by a still-running action is treated
as an ambiguous external outcome: the old action becomes ``unknown``
and the caller receives ``target_reconciliation_required`` rather than
being allowed to write over it.
"""
# Pre-Phase-5 actions have no server-derived target key. Preserve the
# original conditional UPDATE CAS for them; replacing it with a
# read-then-write transaction would weaken their existing cross-worker
# at-most-once guarantee on SQLite (where FOR UPDATE is a no-op).
existing = await self.actions.get_action(action_id)
if existing is None:
raise MafActionStateMachineError(f"action {action_id} not found during claim")
if not existing.get("target_key"):
return await self.actions.claim_for_execution(action_id)
now = _now()
expiry = now + timedelta(seconds=max(1, lease_seconds))
last_error: Exception | None = None
for _attempt in range(4):
try:
async with self._sf() as db:
async with db.begin():
action = (
await db.execute(
select(MafRoundtableActionRow)
.where(MafRoundtableActionRow.id == action_id)
.with_for_update()
)
).scalars().first()
if action is None:
raise MafActionStateMachineError(
f"action {action_id} not found during claim"
)
if action.status == "running":
return "in_progress"
if action.status in ("succeeded", "failed", "unknown", "rejected"):
return str(action.status)
if action.status != "approved":
raise MafActionStateMachineError(
f"action {action_id} cannot be claimed from state '{action.status}'"
)
# Legacy rows can be executed at most once with the old
# action CAS. All new actions receive a target key from
# ActionCoordinator; never manufacture one from client
# input here.
if action.target_key:
lease = (
await db.execute(
select(MafRoundtableActionLeaseRow)
.where(MafRoundtableActionLeaseRow.target_key == action.target_key)
.with_for_update()
)
).scalars().first()
if lease is not None and lease.action_id != action.id:
owner = (
await db.execute(
select(MafRoundtableActionRow)
.where(MafRoundtableActionRow.id == lease.action_id)
.with_for_update()
)
).scalars().first()
if owner is not None and owner.status == "unknown":
# Unknown means the remote side effect may
# have happened. Keep the lease as a
# durable manual-reconciliation barrier;
# another write to the same object must not
# silently leapfrog it.
return "target_reconciliation_required"
if lease.lease_expires_at > now:
return "target_busy"
if owner is not None and owner.status == "running":
owner.status = "unknown"
owner.result_redacted_json = {
"error_code": "target_lease_expired_during_execution"
}
owner.finished_at = now
owner.updated_at = now
# Retain the lease as an explicit manual
# reconciliation barrier. It is released
# only by a future audited reconciliation
# flow, never by automatic replay.
lease.lease_expires_at = now
lease.updated_at = now
return "target_reconciliation_required"
await db.delete(lease)
elif lease is not None and lease.action_id == action.id:
# This can only occur after a failed local
# terminal-result write. Do not treat it as a
# new execution opportunity.
return "in_progress"
db.add(MafRoundtableActionLeaseRow(
target_key=action.target_key,
action_id=action.id,
lease_expires_at=expiry,
created_at=now,
updated_at=now,
))
action.status = "running"
action.attempt_count = int(action.attempt_count or 0) + 1
action.started_at = now
action.updated_at = now
await db.flush()
return "claimed"
except IntegrityError as exc:
# The target-key primary key is the cross-worker arbitration
# point. On a collision retry and observe the winning lease.
last_error = exc
continue
raise RuntimeError(f"could not claim action lease for {action_id}") from last_error
async def record_action_terminal_result(
self,
action_id: str,
*,
status: str,
result: dict[str, Any] | None = None,
external_object_id: str | None = None,
) -> dict[str, Any] | None:
"""Persist a terminal result and release a known target lease atomically.
``unknown`` is intentionally different: its target lease remains as a
manual-reconciliation barrier because the remote side effect may have
completed even though the local process cannot prove it.
"""
if status not in ("succeeded", "failed", "unknown"):
raise ValueError(f"invalid action result status: {status}")
now = _now()
async with self._sf() as db:
async with db.begin():
action = (
await db.execute(
select(MafRoundtableActionRow)
.where(MafRoundtableActionRow.id == action_id)
.with_for_update()
)
).scalars().first()
if action is None or action.status != "running":
return None
action.status = status
action.result_redacted_json = result
action.external_object_id = external_object_id
action.finished_at = now
action.updated_at = now
if status != "unknown":
# A known success or known failure has a deterministic
# outcome, so the next approved action may acquire the
# target. ``unknown`` deliberately retains the barrier.
await db.execute(
delete(MafRoundtableActionLeaseRow).where(
MafRoundtableActionLeaseRow.action_id == action_id
)
)
await db.flush()
return action_to_dict(action)
async def expire_due_approvals(
self, *, run_id: str | None = None, now: datetime | None = None
) -> list[dict[str, Any]]:
"""Expire due approvals and reject unexecuted actions with audit events.
Expiry is deliberately a fail-closed transition: only an action still
waiting for approval becomes rejected; a later/running action is never
overwritten. The approval, action and critical event are committed in
one transaction so polling sees a coherent terminal decision.
"""
current = now or _now()
last_error: Exception | None = None
for _attempt in range(4):
try:
async with self._sf() as db:
async with db.begin():
stmt = select(MafRoundtableApprovalRow).where(
MafRoundtableApprovalRow.status == "pending",
MafRoundtableApprovalRow.expires_at.is_not(None),
MafRoundtableApprovalRow.expires_at <= current,
)
if run_id is not None:
stmt = stmt.where(MafRoundtableApprovalRow.run_id == run_id)
approvals = (await db.execute(stmt.with_for_update())).scalars().all()
if not approvals:
return []
max_seq_by_run: dict[str, int] = {}
expired: list[dict[str, Any]] = []
for approval in approvals:
approval.status = "expired"
approval.decided_by = "system:approval_timeout"
approval.decided_at = current
approval.updated_at = current
action = None
if approval.action_id:
action = (
await db.execute(
select(MafRoundtableActionRow)
.where(MafRoundtableActionRow.id == approval.action_id)
.with_for_update()
)
).scalars().first()
if action is not None and action.status == "waiting_approval":
action.status = "rejected"
action.finished_at = current
action.updated_at = current
if approval.run_id and action is not None:
seq = max_seq_by_run.get(approval.run_id)
if seq is None:
seq = int((
await db.execute(
select(func.max(MafRoundtableEventRow.seq)).where(
MafRoundtableEventRow.run_id == approval.run_id
)
)
).scalar_one() or 0)
seq += 1
max_seq_by_run[approval.run_id] = seq
db.add(MafRoundtableEventRow(
id=uuid4().hex,
session_id=approval.session_id,
run_id=approval.run_id,
seq=seq,
event_type="action.rejected",
agent_id=approval.requested_by_agent_id,
turn_index=action.turn_index,
source_event_id=f"approval-expired:{approval.id}",
payload_json={
"action_id": action.id,
"approval_id": approval.id,
"reason": "approval_expired",
},
created_at=current,
))
expired.append({
"id": approval.id,
"run_id": approval.run_id,
"action_id": approval.action_id,
})
await db.flush()
return expired
except IntegrityError as exc:
last_error = exc
continue
raise RuntimeError("could not persist expired MAF approvals") from last_error
async def recover_expired_action_leases(
self, *, run_id: str, now: datetime | None = None
) -> list[dict[str, Any]]:
"""Mark stale in-flight writes ``unknown`` during restart recovery.
No tool is re-run. A lease that survived a process death cannot prove
whether the remote side effect completed, so the only safe recovery is
an explicit human reconciliation path.
"""
current = now or _now()
async with self._sf() as db:
async with db.begin():
rows = (
await db.execute(
select(MafRoundtableActionLeaseRow, MafRoundtableActionRow)
.join(
MafRoundtableActionRow,
MafRoundtableActionLeaseRow.action_id == MafRoundtableActionRow.id,
)
.where(
MafRoundtableActionRow.run_id == run_id,
MafRoundtableActionLeaseRow.lease_expires_at <= current,
)
.with_for_update()
)
).all()
recovered: list[dict[str, Any]] = []
for lease, action in rows:
if action.status == "running":
action.status = "unknown"
action.result_redacted_json = {
"error_code": "target_lease_expired_during_execution"
}
action.finished_at = current
action.updated_at = current
recovered.append(action_to_dict(action))
# Keep a durable block for manual reconciliation;
# automatically deleting it would allow a later write
# to overtake an ambiguous external side effect.
lease.lease_expires_at = current
lease.updated_at = current
elif action.status != "unknown":
await db.delete(lease)
await db.flush()
return recovered
__all__ = [
"MafActionRepository",
@@ -37,6 +37,7 @@ passing all secret-derived params and the stored action record as plain data.
from __future__ import annotations
import json
import logging
import uuid
from typing import Any, TypedDict
@@ -477,6 +478,15 @@ def _interpret_tool_node_result(result: Any, tool_name: str) -> dict[str, Any]:
if _is_inconclusive_text(text):
return {"status": "unknown", "error": _ERROR_CODE_INCONCLUSIVE}
return {"status": "failed", "error": _ERROR_CODE_FAILED}
# LangChain may serialize a structured tool return into a JSON string
# inside ToolMessage.content. Recover only the safe, allow-listed
# fields so an external object id can be audited without exposing
# arbitrary tool output.
if isinstance(content, str):
try:
content = json.loads(content)
except (TypeError, ValueError):
pass
return {"status": "succeeded", "result": _safe_success_payload(content)}
return {"status": "succeeded", "result": {"executed": True}}
@@ -0,0 +1,77 @@
"""Test-only real external-write tool for MAF approval E2E tests.
This module is deliberately not registered in ``config.yaml``. Tests add the
tool to an isolated ``AppConfig`` and set ``MAF_E2E_OUTBOX_DIR`` to a temporary
directory. It is a real LangChain ``StructuredTool`` and therefore exercises
the same LangGraph ``ToolNode`` path as a configured production tool without
touching a business system.
"""
from __future__ import annotations
import json
import os
import re
import threading
from pathlib import Path
from langchain.tools import tool
_OPERATION_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,95}$")
_TEST_BLOCK_STARTED: threading.Event | None = None
_TEST_BLOCK_RELEASE: threading.Event | None = None
def configure_test_block(
*, started: threading.Event | None = None, release: threading.Event | None = None
) -> None:
"""Install a test-only barrier for lease contention tests.
The hook is inert unless a test explicitly supplies both events. It keeps
the write inside the real synchronous ToolNode call while allowing the
second MAF run to observe the first target lease as ``running``.
"""
global _TEST_BLOCK_STARTED, _TEST_BLOCK_RELEASE
_TEST_BLOCK_STARTED = started
_TEST_BLOCK_RELEASE = release
def clear_test_block() -> None:
configure_test_block()
@tool("maf_test_outbox_write", parse_docstring=True)
def maf_test_outbox_write(operation_id: str, value: str) -> dict[str, object]:
"""Append one controlled write record to the test outbox.
Args:
operation_id: Stable idempotency identifier for the test action.
value: Test payload written only to the temporary outbox.
"""
outbox_dir = (os.getenv("MAF_E2E_OUTBOX_DIR") or "").strip()
if not outbox_dir:
raise RuntimeError("MAF_E2E_OUTBOX_DIR is not configured")
if not _OPERATION_ID.fullmatch(operation_id):
raise ValueError("invalid test outbox operation id")
if _TEST_BLOCK_STARTED is not None and _TEST_BLOCK_RELEASE is not None:
_TEST_BLOCK_STARTED.set()
_TEST_BLOCK_RELEASE.wait()
directory = Path(outbox_dir).resolve()
directory.mkdir(parents=True, exist_ok=True)
record_path = directory / f"{operation_id}.jsonl"
with record_path.open("a", encoding="utf-8", newline="") as record_file:
record_file.write(
json.dumps({"operation_id": operation_id, "value": value}, ensure_ascii=False)
+ "\n"
)
return {
"executed": True,
"external_object_id": operation_id,
"safe_summary": "test outbox write completed",
}
__all__ = ["clear_test_block", "configure_test_block", "maf_test_outbox_write"]
@@ -11,6 +11,7 @@ under ``confirm_high_risk``).
from __future__ import annotations
from collections.abc import Mapping
from typing import Any
_DEFAULT_SIDE_EFFECT = "write"
@@ -42,15 +43,52 @@ def resolve_tool_metadata(
tool_name: str,
*,
config_extra: dict[str, Any] | None = None,
) -> dict[str, str]:
"""Return ``{name, tool_name, side_effect, risk_level}`` for one tool."""
maf_metadata: Mapping[str, Any] | None = None,
) -> dict[str, Any]:
"""Return trusted metadata for one tool.
Precedence is intentionally server-only and explicit:
1. MAF deployment metadata keyed by the exact runtime tool name;
2. existing server ``ToolConfig`` extras;
3. the curated built-in allow-list;
4. conservative ``write`` / ``R3`` fallback.
``maf_metadata`` is passed by the application while building a frozen
capability snapshot. It is never accepted from a session request or a
model message.
"""
side = _DEFAULT_SIDE_EFFECT
risk = _DEFAULT_RISK_LEVEL
if config_extra:
if config_extra.get("side_effect"):
side = str(config_extra["side_effect"])
if config_extra.get("risk_level"):
risk = str(config_extra["risk_level"])
target_key_fields: list[str] = []
target_key_namespace = "default"
registered = maf_metadata.get(tool_name) if maf_metadata else None
if registered is not None:
candidate_side = registered.get("side_effect") if isinstance(registered, Mapping) else getattr(registered, "side_effect", None)
candidate_risk = registered.get("risk_level") if isinstance(registered, Mapping) else getattr(registered, "risk_level", None)
if candidate_side in {"read", "skill", "write"}:
side = str(candidate_side)
if candidate_risk in {"R0", "R1", "R2", "R3", "R4"}:
risk = str(candidate_risk)
candidate_fields = registered.get("target_key_fields") if isinstance(registered, Mapping) else getattr(registered, "target_key_fields", None)
candidate_namespace = registered.get("target_key_namespace") if isinstance(registered, Mapping) else getattr(registered, "target_key_namespace", None)
if isinstance(candidate_fields, list) and all(isinstance(item, str) for item in candidate_fields):
target_key_fields = list(candidate_fields)
if isinstance(candidate_namespace, str) and candidate_namespace:
target_key_namespace = candidate_namespace
elif config_extra:
candidate_side = config_extra.get("side_effect")
candidate_risk = config_extra.get("risk_level")
if candidate_side in {"read", "skill", "write"}:
side = str(candidate_side)
if candidate_risk in {"R0", "R1", "R2", "R3", "R4"}:
risk = str(candidate_risk)
candidate_fields = config_extra.get("target_key_fields")
candidate_namespace = config_extra.get("target_key_namespace")
if isinstance(candidate_fields, list) and all(isinstance(item, str) for item in candidate_fields):
target_key_fields = list(candidate_fields)
if isinstance(candidate_namespace, str) and candidate_namespace:
target_key_namespace = candidate_namespace
elif tool_name in _BUILTIN_METADATA:
side, risk = _BUILTIN_METADATA[tool_name]
return {
@@ -58,4 +96,6 @@ def resolve_tool_metadata(
"tool_name": tool_name,
"side_effect": side,
"risk_level": risk,
"target_key_fields": target_key_fields,
"target_key_namespace": target_key_namespace,
}
@@ -0,0 +1,139 @@
"""Phase 5 migration for MAF external-write target leases.
Creates the additive ``maf_roundtable_action_leases`` table used to serialize
approved writes to the same server-derived external business target. Existing
action rows are intentionally not backfilled: old rows have no trusted target
key and retain the legacy action-level at-most-once behavior. New writes
receive an opaque target key from ``ActionCoordinator``.
Run this before deploying code that enables MAF external writes:
PYTHONPATH=. uv run python scripts/migrate_maf_roundtable_phase5.py --dry-run
PYTHONPATH=. uv run python scripts/migrate_maf_roundtable_phase5.py
"""
from __future__ import annotations
import argparse
import asyncio
import logging
import sys
from typing import Any
logging.basicConfig(level=logging.INFO, format="%(message)s")
logger = logging.getLogger("migrate_maf_roundtable_phase5")
TABLE = "maf_roundtable_action_leases"
ACTION_INDEX = "ix_maf_action_leases_action_id"
EXPIRY_INDEX = "ix_maf_action_leases_lease_expires_at"
def _dialect_name(conn_or_dialect: Any) -> str:
return str(getattr(getattr(conn_or_dialect, "dialect", conn_or_dialect), "name", "sqlite")).lower()
def _timestamp_sql(conn_or_dialect: Any) -> str:
"""Return the timestamp type accepted by each supported database."""
dialect = _dialect_name(conn_or_dialect)
if dialect == "postgresql":
return "TIMESTAMP WITH TIME ZONE"
if dialect in {"mysql", "mariadb"}:
return "DATETIME(6)"
return "DATETIME"
def _create_lease_table_sql(conn_or_dialect: Any) -> str:
timestamp_type = _timestamp_sql(conn_or_dialect)
return f"""
CREATE TABLE {TABLE} (
target_key VARCHAR(255) NOT NULL PRIMARY KEY,
action_id VARCHAR(64) NOT NULL,
lease_expires_at {timestamp_type} NOT NULL,
created_at {timestamp_type} NOT NULL,
updated_at {timestamp_type} NOT NULL,
CONSTRAINT uq_maf_action_leases_action UNIQUE (action_id),
FOREIGN KEY (action_id) REFERENCES maf_roundtable_actions (id) ON DELETE CASCADE
)
"""
def _add_missing_columns(conn: Any, *, inspector: Any, execute: Any) -> None:
"""Complete a partially created table without rewriting existing data.
The migration is additive. A missing primary key or action foreign-key
column cannot be safely added to a populated table, so fail loudly rather
than creating a lease table that the arbiter could misinterpret.
"""
columns = {column["name"] for column in inspector.get_columns(TABLE)}
required = {"target_key", "action_id", "lease_expires_at", "created_at", "updated_at"}
missing = required - columns
if not missing:
return
if {"target_key", "action_id"} & missing:
raise RuntimeError(
f"{TABLE} is incomplete ({sorted(missing)}); recreate it with the deployment migration"
)
timestamp_type = _timestamp_sql(conn)
for column in ("lease_expires_at", "created_at", "updated_at"):
if column in missing:
execute(f"ALTER TABLE {TABLE} ADD COLUMN {column} {timestamp_type}")
async def _migrate(dry_run: bool) -> int:
from deerflow.config import get_app_config
from deerflow.persistence.engine import close_engine, get_session_factory, init_engine_from_config
await init_engine_from_config(get_app_config().database)
sf = get_session_factory()
if sf is None:
logger.error("Database backend is 'memory' — nothing to migrate.")
return 0
engine = sf.kw["bind"]
async with engine.begin() as conn:
await conn.run_sync(_migrate_sync, dry_run=dry_run)
await close_engine()
return 0
def _migrate_sync(conn: Any, *, dry_run: bool) -> None:
from sqlalchemy import inspect, text
inspector = inspect(conn)
def execute(sql: str) -> None:
if dry_run:
logger.info("[dry-run] would execute: %s", " ".join(sql.split()))
return
conn.execute(text(sql))
if TABLE not in inspector.get_table_names():
logger.info("Creating %s", TABLE)
execute(_create_lease_table_sql(conn))
if dry_run:
execute(f"CREATE INDEX {ACTION_INDEX} ON {TABLE} (action_id)")
execute(f"CREATE INDEX {EXPIRY_INDEX} ON {TABLE} (lease_expires_at)")
return
inspector = inspect(conn)
else:
logger.info("Table %s already exists", TABLE)
_add_missing_columns(conn, inspector=inspector, execute=execute)
if not dry_run:
inspector = inspect(conn)
indexes = {index.get("name") for index in inspector.get_indexes(TABLE)}
if ACTION_INDEX not in indexes:
execute(f"CREATE INDEX {ACTION_INDEX} ON {TABLE} (action_id)")
if EXPIRY_INDEX not in indexes:
execute(f"CREATE INDEX {EXPIRY_INDEX} ON {TABLE} (lease_expires_at)")
def main() -> int:
parser = argparse.ArgumentParser(description="Create MAF external-write target lease table")
parser.add_argument("--dry-run", action="store_true", help="Preview only; do not write")
return asyncio.run(_migrate(parser.parse_args().dry_run))
if __name__ == "__main__":
sys.exit(main())
@@ -0,0 +1,396 @@
"""Phase-5 target lease, timeout and ambiguous-result safety tests."""
from __future__ import annotations
import asyncio
from datetime import UTC, datetime, timedelta
from types import SimpleNamespace
import pytest
import pytest_asyncio
from maf_test_helpers import make_maf_service
from sqlalchemy import update
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from app.gateway.app import _scan_maf_approval_expiry
from app.maf_roundtable.action_coordinator import (
ActionCoordinator,
derive_target_key,
encrypt_params,
)
from app.maf_roundtable.service import MafConflictError
from app.maf_roundtable.tool_executor import ToolExecutor
from app.maf_roundtable.workflow_executors import RoundtableState
from app.maf_roundtable.workflow_runner import WorkflowRunner
from deerflow.config.maf_roundtable_config import MafRoundtableConfig
from deerflow.persistence.maf_roundtable import make_maf_roundtable_store
from deerflow.persistence.maf_roundtable.model import (
Base,
MafRoundtableActionLeaseRow,
MafRoundtableApprovalRow,
)
TEST_TOOL = "controlled_write"
def _snapshot() -> dict:
tool = {
"name": TEST_TOOL,
"side_effect": "write",
"risk_level": "R3",
"target_key_fields": ["tenant_id", "object_id"],
"target_key_namespace": "lease-test",
}
return {
"tools": [tool],
"agents": [{"agent_id": "seat-1", "tools": [tool]}],
}
@pytest_asyncio.fixture
async def store(tmp_path):
# A file-backed SQLite database gives concurrent sessions the same
# transaction/constraint semantics used by deployed SQLite installations.
engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'maf_action_leases.db'}", echo=False)
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
sf = async_sessionmaker(engine, expire_on_commit=False)
result = make_maf_roundtable_store(sf)
assert result is not None
yield result
await engine.dispose()
@pytest.fixture
def svc(store):
return make_maf_service(
store,
enabled=True,
config=MafRoundtableConfig(
enabled=True,
approval_timeout_seconds=60,
target_lease_seconds=30,
),
)
async def _run(store, svc) -> tuple[dict, dict]:
session = await svc.create_session(user_id="u1", body={"task_text": "test"})
run = await svc.create_run(session_id=session["id"], user_id="u1")
return session, run
async def _approved_action(
store, session: dict, run: dict, *, key: str, target_key: str, agent_id: str = "seat-1"
) -> dict:
return await store.actions.create_action({
"session_id": session["id"],
"run_id": run["id"],
"agent_id": agent_id,
"tool_name": TEST_TOOL,
"idempotency_key": key,
"target_key": target_key,
"risk_level": "R3",
"status": "approved",
})
def test_target_key_is_server_derived_and_opaque(monkeypatch) -> None:
snapshot = _snapshot()
original = {
"tenant_id": "tenant-a",
"object_id": "task-7",
"target_key": "browser-forged-value",
"secret": "do-not-persist",
}
same_target_different_noise = {
"tenant_id": "tenant-a",
"object_id": "task-7",
"target_key": "another-forged-value",
"secret": "changed",
}
target = derive_target_key(TEST_TOOL, original, snapshot, "seat-1")
assert target == derive_target_key(TEST_TOOL, same_target_different_noise, snapshot, "seat-1")
assert target.startswith("maf-target:v1:")
assert "tenant-a" not in target
assert "task-7" not in target
assert "browser-forged" not in target
nested = derive_target_key(
TEST_TOOL,
{"tenant_id": "tenant-a", "object_id": {"id": "task-7"}},
snapshot,
"seat-1",
)
nested_other = derive_target_key(
TEST_TOOL,
{"tenant_id": "tenant-a", "object_id": ["task-7"]},
snapshot,
"seat-1",
)
assert nested == nested_other
monkeypatch.setenv("MAF_ACTION_SECRET_KEY", "another-test-domain-secret")
assert target != derive_target_key(TEST_TOOL, original, snapshot, "seat-1")
@pytest.mark.asyncio
async def test_two_agents_same_target_are_serialized(store, svc):
"""The DB lease, not a process lock, blocks the second approved write."""
session, run = await _run(store, svc)
target_key = derive_target_key(
TEST_TOOL, {"tenant_id": "tenant-a", "object_id": "task-7"}, _snapshot(), "seat-1"
)
first = await _approved_action(store, session, run, key="lease-first", target_key=target_key)
second = await _approved_action(
store, session, run, key="lease-second", target_key=target_key, agent_id="seat-2"
)
first_claim, second_claim = await asyncio.gather(
store.claim_action_for_execution(first["id"], lease_seconds=30),
store.claim_action_for_execution(second["id"], lease_seconds=30),
)
assert sorted([first_claim, second_claim]) == ["claimed", "target_busy"]
winner = first if first_claim == "claimed" else second
waiting = second if winner["id"] == first["id"] else first
assert (await store.actions.get_action(waiting["id"]))["status"] == "approved"
assert await store.record_action_terminal_result(
winner["id"], status="succeeded", result={"safe": True}
)
assert await store.claim_action_for_execution(waiting["id"], lease_seconds=30) == "claimed"
@pytest.mark.asyncio
async def test_restart_recovers_expired_lease_as_unknown_without_replay(store, svc):
session, run = await _run(store, svc)
target_key = derive_target_key(
TEST_TOOL, {"tenant_id": "tenant-a", "object_id": "task-7"}, _snapshot(), "seat-1"
)
action = await _approved_action(store, session, run, key="lease-restart", target_key=target_key)
assert await store.claim_action_for_execution(action["id"], lease_seconds=1) == "claimed"
recovered = await store.recover_expired_action_leases(
run_id=run["id"], now=datetime.now(UTC) + timedelta(seconds=5)
)
assert [item["id"] for item in recovered] == [action["id"]]
stored = await store.actions.get_action(action["id"])
assert stored["status"] == "unknown"
assert stored["result"] == {"error_code": "target_lease_expired_during_execution"}
assert await store.claim_action_for_execution(action["id"], lease_seconds=30) == "unknown"
blocked = await _approved_action(
store,
session,
run,
key="lease-restart-blocked",
target_key=target_key,
agent_id="seat-2",
)
assert await store.claim_action_for_execution(blocked["id"], lease_seconds=30) == "target_reconciliation_required"
@pytest.mark.asyncio
async def test_approval_timeout_rejects_action_and_writes_audit_event(monkeypatch, store, svc):
monkeypatch.setenv("MAF_ACTION_SECRET_KEY", "test-only-lease-secret")
session, run = await _run(store, svc)
coordinator = ActionCoordinator(svc, user_id="u1")
actions = await coordinator.intercept_write_tool_calls(
run_id=run["id"],
session_id=session["id"],
agent_id="seat-1",
turn_index=1,
tool_calls=[{
"name": TEST_TOOL,
"id": "timeout-call",
"args": {"tenant_id": "tenant-a", "object_id": "task-7"},
}],
capability_snapshot=_snapshot(),
write_policy="confirm_all_writes",
)
approval_id = actions[0]["approval_id"]
expired = await store.expire_due_approvals(
run_id=run["id"], now=datetime.now(UTC) + timedelta(minutes=2)
)
assert expired == [{"id": approval_id, "run_id": run["id"], "action_id": actions[0]["id"]}]
assert (await store.approvals.get_approval(approval_id))["status"] == "expired"
assert (await store.actions.get_action(actions[0]["id"]))["status"] == "rejected"
with pytest.raises(MafConflictError):
await svc.decide_approval(approval_id=approval_id, decision="approved", user_id="u1")
events = await store.events.list_events(run["id"])
assert any(
event["event_type"] == "action.rejected"
and event["payload"].get("reason") == "approval_expired"
for event in events
)
@pytest.mark.asyncio
async def test_external_success_with_local_result_failure_never_replays(monkeypatch, store, svc):
"""After external success, an unavailable DB result write becomes unknown."""
monkeypatch.setenv("MAF_ACTION_SECRET_KEY", "test-only-lease-secret")
session, run = await _run(store, svc)
target_key = derive_target_key(
TEST_TOOL, {"tenant_id": "tenant-a", "object_id": "task-7"}, _snapshot(), "seat-1"
)
action = await _approved_action(store, session, run, key="persist-failure", target_key=target_key)
await store.action_secrets.create(
action_id=action["id"], arguments_encrypted=encrypt_params({"tenant_id": "tenant-a", "object_id": "task-7"})
)
calls = 0
class Client:
async def execute_tool_directly(self, **_kwargs):
nonlocal calls
calls += 1
return {"status": "succeeded", "result": {"executed": True}}
adapter = SimpleNamespace(_client=Client())
coordinator = ActionCoordinator(svc, user_id="u1")
async def fail_terminal_persistence(*_args, **_kwargs):
raise RuntimeError("simulated local persistence outage")
monkeypatch.setattr(coordinator, "record_result", fail_terminal_persistence)
executor = ToolExecutor(adapter=adapter, action_coordinator=coordinator, user_id="u1")
first = await executor.execute(
action_id=action["id"],
session_id=session["id"],
run_id=run["id"],
agent_id="seat-1",
thread_id="thread-1",
approved_tool=TEST_TOOL,
capability_snapshot=_snapshot(),
write_policy="confirm_all_writes",
)
second = await executor.execute(
action_id=action["id"],
session_id=session["id"],
run_id=run["id"],
agent_id="seat-1",
thread_id="thread-1",
approved_tool=TEST_TOOL,
capability_snapshot=_snapshot(),
write_policy="confirm_all_writes",
)
assert first["status"] == "unknown"
assert second["status"] == "in_progress"
assert calls == 1
recovered = await store.recover_expired_action_leases(
run_id=run["id"], now=datetime.now(UTC) + timedelta(minutes=20)
)
assert [item["id"] for item in recovered] == [action["id"]]
assert (await store.actions.get_action(action["id"]))["status"] == "unknown"
@pytest.mark.asyncio
async def test_workflow_recovery_stops_on_expired_lease_without_tool_replay(store, svc):
"""The restart entry point converts a stale running lease to waiting_input."""
session, run = await _run(store, svc)
target_key = derive_target_key(
TEST_TOOL, {"tenant_id": "tenant-a", "object_id": "task-7"}, _snapshot(), "seat-1"
)
action = await _approved_action(store, session, run, key="runner-recovery", target_key=target_key)
assert await store.claim_action_for_execution(action["id"], lease_seconds=30) == "claimed"
async with store.session_factory() as db:
await db.execute(
update(MafRoundtableActionLeaseRow)
.where(MafRoundtableActionLeaseRow.action_id == action["id"])
.values(lease_expires_at=datetime.now(UTC) - timedelta(seconds=1))
)
await db.commit()
state = RoundtableState(
session_id=session["id"],
run_id=run["id"],
task_text="test",
mode="structured",
model_name=None,
reasoning_effort=None,
max_rounds=1,
write_policy="confirm_all_writes",
manager_agent_id=None,
seat_agent_ids=[],
thread_map={},
capability_snapshot=_snapshot(),
)
state.current_round = 1
state.status = "running"
state.pending_approvals = [{
"action_id": action["id"],
"approval_id": "already-approved",
"agent_id": "seat-1",
"tool_name": TEST_TOOL,
}]
await store.runs.update_run(run["id"], {"status": "running", "maf_checkpoint": state.to_dict()})
await store.sessions.update_session(session["id"], user_id="u1", fields={"status": "running"})
runner = WorkflowRunner(SimpleNamespace(), service=svc, user_id="u1")
recovered = await runner.recover(run["id"])
assert recovered is not None and recovered["status"] == "waiting_input"
stored = await store.actions.get_action(action["id"])
assert stored["status"] == "unknown"
assert (await store.sessions.get_session(session["id"], user_id="u1"))["status"] == "waiting_input"
@pytest.mark.asyncio
async def test_approval_expiry_scan_is_repeatable_and_get_is_observational(monkeypatch, store, svc):
"""A scan expires once; ordinary approval GETs never perform that mutation."""
session, run = await _run(store, svc)
coordinator = ActionCoordinator(svc, user_id="u1")
action = (
await coordinator.intercept_write_tool_calls(
run_id=run["id"],
session_id=session["id"],
agent_id="seat-1",
turn_index=1,
tool_calls=[{"name": TEST_TOOL, "id": "expiry-scan", "args": {
"tenant_id": "tenant-a", "object_id": "task-7", "value": "x"
}}],
capability_snapshot=_snapshot(),
write_policy="confirm_all_writes",
)
)[0]
async with store.session_factory() as db:
await db.execute(
update(MafRoundtableApprovalRow)
.where(MafRoundtableApprovalRow.id == action["approval_id"])
.values(expires_at=datetime.now(UTC) - timedelta(seconds=1))
)
await db.commit()
# The normal read path must not mutate an expired pending row.
listed = await svc.list_approvals(run_id=run["id"], user_id="u1")
assert listed[0]["status"] == "pending"
class FakeDispatcher:
def __init__(self):
self.calls: list[tuple[str, str]] = []
async def resume(self, run_id: str, *, user_id: str):
self.calls.append((run_id, user_id))
return await store.runs.get_run(run_id)
fake = FakeDispatcher()
monkeypatch.setattr(
"app.maf_roundtable.dispatcher.get_dispatcher",
lambda _app: fake,
)
app = SimpleNamespace(
state=SimpleNamespace(maf_roundtable_service=svc),
)
concurrent = await asyncio.gather(
_scan_maf_approval_expiry(app),
_scan_maf_approval_expiry(app),
)
repeat_after_restart = await _scan_maf_approval_expiry(app)
assert sorted(concurrent, key=len) == [[], [run["id"]]]
assert repeat_after_restart == []
assert fake.calls == [(run["id"], "u1")]
assert (await store.approvals.get_approval(action["approval_id"]))["status"] == "expired"
assert (await store.actions.get_action(action["id"]))["status"] == "rejected"
@@ -50,7 +50,11 @@ CHANNEL_VALUES: dict[str, Any] = {
def app_config():
from deerflow.config import get_app_config
return get_app_config()
# The production config mounts WriteGuard. These low-level executor tests
# that do not register a MAF service intentionally exercise the harness
# path without the app-layer validator; the dedicated enabled tests below
# provide the real bridge/service wiring.
return get_app_config().model_copy(update={"write_guard": WriteGuardConfig(enabled=False)})
def _snapshot(action_tool: str) -> dict[str, Any]:
@@ -86,3 +86,33 @@ async def test_inaccessible_agent_rejected():
app_config=app_config,
agent_store=_DenyStore(),
)
def test_maf_tool_metadata_has_highest_server_priority():
metadata = resolve_tool_metadata(
"weknora-retrieval_weknora_search",
config_extra={"side_effect": "write", "risk_level": "R4"},
maf_metadata={"weknora-retrieval_weknora_search": {"side_effect": "read", "risk_level": "R0"}},
)
assert metadata["side_effect"] == "read"
assert metadata["risk_level"] == "R0"
def test_deployed_weknora_names_are_explicitly_read_only():
metadata = get_app_config().maf_roundtable.tool_metadata
assert metadata["weknora-retrieval_weknora_list_knowledge_bases"].side_effect == "read"
assert metadata["weknora-retrieval_weknora_list_knowledge_bases"].risk_level == "R0"
assert metadata["weknora-retrieval_weknora_search"].side_effect == "read"
assert metadata["weknora-retrieval_weknora_search"].risk_level == "R0"
def test_unknown_tool_metadata_remains_conservative():
metadata = resolve_tool_metadata(
"unregistered_external_tool",
config_extra={"side_effect": "invalid", "risk_level": "R99"},
)
assert metadata["side_effect"] == "write"
assert metadata["risk_level"] == "R3"
@@ -0,0 +1,89 @@
"""Acceptance coverage for the additive Phase-5 target-lease migration."""
from __future__ import annotations
import importlib.util
from pathlib import Path
import pytest
from sqlalchemy import inspect, text
from sqlalchemy.ext.asyncio import create_async_engine
from deerflow.persistence.maf_roundtable.model import Base
def _migration_module():
script = Path(__file__).resolve().parent.parent / "scripts" / "migrate_maf_roundtable_phase5.py"
spec = importlib.util.spec_from_file_location("migrate_maf_roundtable_phase5", script)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
@pytest.mark.asyncio
async def test_phase5_lease_migration_is_additive_and_idempotent():
module = _migration_module()
engine = create_async_engine("sqlite+aiosqlite://", echo=False)
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
await conn.execute(text("DROP TABLE maf_roundtable_action_leases"))
async with engine.begin() as conn:
await conn.run_sync(module._migrate_sync, dry_run=False)
def check(sync_conn):
inspector = inspect(sync_conn)
assert "maf_roundtable_action_leases" in inspector.get_table_names()
columns = {column["name"] for column in inspector.get_columns("maf_roundtable_action_leases")}
assert {"target_key", "action_id", "lease_expires_at"} <= columns
indexes = {index["name"] for index in inspector.get_indexes("maf_roundtable_action_leases")}
assert {"ix_maf_action_leases_action_id", "ix_maf_action_leases_lease_expires_at"} <= indexes
async with engine.begin() as conn:
await conn.run_sync(check)
await conn.execute(text(f"DROP INDEX {module.ACTION_INDEX}"))
await conn.execute(text(f"DROP INDEX {module.EXPIRY_INDEX}"))
await conn.run_sync(module._migrate_sync, dry_run=False)
await conn.run_sync(check)
await conn.run_sync(module._migrate_sync, dry_run=True)
await engine.dispose()
@pytest.mark.asyncio
async def test_phase5_migration_dry_run_is_safe_when_table_is_absent():
module = _migration_module()
engine = create_async_engine("sqlite+aiosqlite://", echo=False)
async with engine.begin() as conn:
await conn.run_sync(module._migrate_sync, dry_run=True)
tables = await conn.run_sync(lambda sync_conn: inspect(sync_conn).get_table_names())
assert "maf_roundtable_action_leases" not in tables
await engine.dispose()
def test_phase5_ddl_uses_sqlite_datetime():
module = _migration_module()
ddl = module._create_lease_table_sql(SimpleDialect("sqlite"))
assert "lease_expires_at DATETIME NOT NULL" in ddl
assert "created_at DATETIME NOT NULL" in ddl
def test_phase5_ddl_uses_postgresql_timestamp_with_timezone():
module = _migration_module()
ddl = module._create_lease_table_sql(SimpleDialect("postgresql"))
assert "lease_expires_at TIMESTAMP WITH TIME ZONE NOT NULL" in ddl
assert "created_at TIMESTAMP WITH TIME ZONE NOT NULL" in ddl
assert "DATETIME" not in ddl
def test_phase5_ddl_uses_mysql_microsecond_datetime():
module = _migration_module()
ddl = module._create_lease_table_sql(SimpleDialect("mysql"))
assert "lease_expires_at DATETIME(6) NOT NULL" in ddl
assert "created_at DATETIME(6) NOT NULL" in ddl
class SimpleDialect:
def __init__(self, name: str) -> None:
self.name = name
@@ -0,0 +1,312 @@
"""Entry-level MAF Phase-5 approval and target-lease acceptance tests."""
from __future__ import annotations
import asyncio
import threading
from types import SimpleNamespace
import pytest
from fastapi import FastAPI, Request
from httpx import ASGITransport, AsyncClient
from langchain_core.language_models.chat_models import BaseChatModel
from langchain_core.messages import AIMessage
from langchain_core.outputs import ChatGeneration, ChatResult
from langgraph.checkpoint.memory import InMemorySaver
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from app.gateway.routers.maf_roundtable import router as maf_router
from app.maf_roundtable.dispatcher import get_dispatcher
from app.maf_roundtable.write_guard_bridge import register_write_guard_service
from deerflow.config import get_app_config
from deerflow.config.maf_roundtable_config import MafRoundtableConfig, MafToolMetadata
from deerflow.config.tool_config import ToolConfig
from deerflow.config.write_guard_config import WriteGuardConfig, WriteGuardProviderConfig
from deerflow.persistence.base import Base
from deerflow.persistence.maf_roundtable import make_maf_roundtable_store
from deerflow.runtime import RunManager
from deerflow.runtime.events.store.memory import MemoryRunEventStore
from deerflow.runtime.stream_bridge.memory import MemoryStreamBridge
TEST_TOOL = "maf_test_outbox_write"
class _User:
def __init__(self, user_id: str) -> None:
self.id = user_id
self.system_role = "user"
class _ThreadStore:
def __init__(self) -> None:
self.rows: dict[str, dict] = {}
async def create(self, thread_id, assistant_id=None, metadata=None):
self.rows[thread_id] = {
"thread_id": thread_id,
"assistant_id": assistant_id,
"metadata": metadata or {},
"status": "idle",
}
async def get(self, thread_id):
return self.rows.get(thread_id)
async def update_status(self, thread_id, status):
self.rows.setdefault(thread_id, {})["status"] = status
async def update_assistant_id(self, thread_id, assistant_id):
self.rows.setdefault(thread_id, {})["assistant_id"] = assistant_id
async def update_metadata(self, thread_id, metadata):
self.rows.setdefault(thread_id, {}).setdefault("metadata", {}).update(metadata)
async def update_display_name(self, thread_id, display_name):
self.rows.setdefault(thread_id, {})["display_name"] = display_name
class _DeterministicModel(BaseChatModel):
"""Deterministic model boundary: propose one write, then finish."""
calls: int = 0
@property
def _llm_type(self) -> str:
return "maf-phase5-deterministic"
def bind_tools(self, tools, **kwargs): # noqa: ARG002
return self
def _generate(self, messages, stop=None, run_manager=None, **kwargs): # noqa: ARG002
if self.calls == 0:
message = AIMessage(
content="",
tool_calls=[{
"name": TEST_TOOL,
"args": {"operation_id": "phase5-target", "value": "controlled"},
"id": "phase5-tool-call",
"type": "tool_call",
}],
)
else:
message = AIMessage(content="deterministic completion")
self.calls += 1
return ChatResult(generations=[ChatGeneration(message=message)])
def _app_config():
config = get_app_config().model_copy(deep=True)
config.tools = [
*config.tools,
ToolConfig(
name=TEST_TOOL,
group="maf-e2e",
use="deerflow.tools.builtins.maf_test_outbox:maf_test_outbox_write",
),
]
config.write_guard = WriteGuardConfig(
enabled=True,
provider=WriteGuardProviderConfig(
policy_use="app.maf_roundtable.write_guard_bridge:make_policy",
interceptor_use="app.maf_roundtable.write_guard_bridge:make_interceptor",
),
)
config.maf_roundtable = MafRoundtableConfig(
enabled=True,
default_write_policy="confirm_all_writes",
approval_timeout_seconds=300,
approval_scan_interval_seconds=1,
tool_metadata={
TEST_TOOL: MafToolMetadata(
side_effect="write",
risk_level="R2",
target_key_fields=["operation_id"],
target_key_namespace="phase5-e2e",
)
},
)
return config
class _VisibleAgentStore:
async def get_visible(self, agent_id: str, user_id: str): # noqa: ARG002
return {"id": agent_id, "user_id": user_id}
async def ensure_builtin(self, record): # noqa: ARG002
return None
def _install_test_app(app: FastAPI, service, config) -> None:
app.state.config = config
app.state.maf_roundtable_service = service
app.state.maf_auto_dispatch = True
app.state.checkpointer = InMemorySaver()
app.state.thread_store = _ThreadStore()
app.state.run_manager = RunManager()
app.state.stream_bridge = MemoryStreamBridge()
app.state.run_event_store = MemoryRunEventStore()
app.state.store = None
app.state.agent_store = _VisibleAgentStore()
register_write_guard_service(service)
app.include_router(maf_router)
async def _wait_dispatch(dispatcher, run_id: str) -> None:
task = dispatcher._tasks.get(run_id) # noqa: SLF001 - entry test waits on the real task
assert task is not None
await task
@pytest.fixture
def deterministic_agent(monkeypatch, tmp_path):
agent_config = SimpleNamespace(
tool_groups=["maf-e2e"],
id="seat-1",
description="MAF E2E test seat",
model=None,
skills=[],
subagent_enabled=False,
name="MAF E2E seat",
)
paths = (
"deerflow.config.agents_config.load_agent_config",
"app.maf_roundtable.capability_snapshot.load_agent_config",
"deerflow.agents.lead_agent.agent.load_agent_config",
)
for path in paths:
monkeypatch.setattr(path, lambda _agent_id, cfg=agent_config: cfg)
monkeypatch.setattr("deerflow.agents.lead_agent.agent.load_agent_soul", lambda _agent_id: "")
monkeypatch.setattr(
"deerflow.agents.lead_agent.agent.create_chat_model",
lambda **_kwargs: _DeterministicModel(),
)
config = _app_config()
config.title.enabled = False
monkeypatch.setattr("deerflow.config.get_app_config", lambda: config)
monkeypatch.setenv("MAF_ACTION_SECRET_KEY", "phase5-entry-test-secret")
monkeypatch.setenv("MAF_E2E_OUTBOX_DIR", str(tmp_path))
return config
async def _make_app(tmp_path, config):
engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'maf_entry.db'}")
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
sf = async_sessionmaker(engine, expire_on_commit=False)
store = make_maf_roundtable_store(sf)
from app.maf_roundtable.service import MafRoundtableService
service = MafRoundtableService(
store,
enabled=True,
admin_diagnostic_access=False,
config=config.maf_roundtable,
agent_store=_VisibleAgentStore(),
app_config=config,
)
app = FastAPI()
_install_test_app(app, service, config)
@app.middleware("http")
async def test_auth(request: Request, call_next):
request.state.user = _User("u1")
return await call_next(request)
return app, service, engine
async def _create_waiting_run(client: AsyncClient, dispatcher, *, task_text: str, service=None) -> tuple[str, dict]:
session_response = await client.post(
"/api/maf-roundtable/sessions",
json={
"task_text": task_text,
"write_policy": "confirm_all_writes",
"participants": [{"agent_id": "seat-1", "role": "seat"}],
},
)
assert session_response.status_code == 200, session_response.text
session = session_response.json()
run_response = await client.post(
f"/api/maf-roundtable/sessions/{session['id']}/runs",
json={},
)
assert run_response.status_code == 200, run_response.text
run = run_response.json()
await _wait_dispatch(dispatcher, run["id"])
run_state = (await client.get(f"/api/maf-roundtable/runs/{run['id']}")).json()
if run_state["status"] != "waiting_approval" and service is not None:
actions = await service.store.actions.list_actions(run_id=run["id"])
raise AssertionError(f"unexpected run state: {run_state}; actions={actions}")
assert run_state["status"] == "waiting_approval"
return session["id"], run
@pytest.mark.asyncio
async def test_entry_http_approval_reaches_real_toolnode_once(tmp_path, deterministic_agent):
app, service, engine = await _make_app(tmp_path, deterministic_agent)
dispatcher = get_dispatcher(app)
try:
transport = ASGITransport(app=app)
async with AsyncClient(transport=transport, base_url="http://test") as client:
session_id, run = await _create_waiting_run(client, dispatcher, task_text="entry approval", service=service)
approvals = (await client.get(f"/api/maf-roundtable/runs/{run['id']}/approvals")).json()["approvals"]
assert len(approvals) == 1
approval_response = await client.post(
f"/api/maf-roundtable/approvals/{approvals[0]['id']}/decision",
json={"decision": "approved"},
)
assert approval_response.status_code == 200, approval_response.text
action = (await client.get(f"/api/maf-roundtable/runs/{run['id']}/actions")).json()["actions"][0]
assert action["status"] == "succeeded"
outbox = tmp_path / "phase5-target.jsonl"
assert outbox.exists()
assert outbox.read_text(encoding="utf-8").count("\n") == 1
assert (await client.get(f"/api/maf-roundtable/sessions/{session_id}")).json()["status"] == "completed"
finally:
await engine.dispose()
@pytest.mark.asyncio
async def test_entry_http_same_target_waits_for_explicit_resume(tmp_path, deterministic_agent):
app, service, engine = await _make_app(tmp_path, deterministic_agent)
dispatcher = get_dispatcher(app)
from deerflow.tools.builtins import maf_test_outbox
started = threading.Event()
release = threading.Event()
maf_test_outbox.configure_test_block(started=started, release=release)
try:
transport = ASGITransport(app=app)
async with AsyncClient(transport=transport, base_url="http://test") as client:
_session_one, run_one = await _create_waiting_run(client, dispatcher, task_text="lease one", service=service)
approval_one = (await client.get(f"/api/maf-roundtable/runs/{run_one['id']}/approvals")).json()["approvals"][0]
first_decision = asyncio.create_task(client.post(
f"/api/maf-roundtable/approvals/{approval_one['id']}/decision",
json={"decision": "approved"},
))
assert await asyncio.to_thread(started.wait, 10)
_session_two, run_two = await _create_waiting_run(client, dispatcher, task_text="lease two", service=service)
approval_two = (await client.get(f"/api/maf-roundtable/runs/{run_two['id']}/approvals")).json()["approvals"][0]
second_decision = await client.post(
f"/api/maf-roundtable/approvals/{approval_two['id']}/decision",
json={"decision": "approved"},
)
assert second_decision.status_code == 200
assert (await client.get(f"/api/maf-roundtable/runs/{run_two['id']}")).json()["status"] == "waiting_approval"
second_action = (await client.get(f"/api/maf-roundtable/runs/{run_two['id']}/actions")).json()["actions"][0]
assert second_action["status"] == "approved"
release.set()
assert (await first_decision).status_code == 200
assert (await client.get(f"/api/maf-roundtable/runs/{run_one['id']}")).json()["status"] == "completed"
resumed = await client.post(f"/api/maf-roundtable/runs/{run_two['id']}/resume")
assert resumed.status_code == 200, resumed.text
assert (await client.get(f"/api/maf-roundtable/runs/{run_two['id']}")).json()["status"] == "completed"
outbox = tmp_path / "phase5-target.jsonl"
assert outbox.read_text(encoding="utf-8").count("\n") == 2
finally:
release.set()
maf_test_outbox.clear_test_block()
await engine.dispose()
@@ -0,0 +1,238 @@
"""MAF approval-to-real-tool E2E coverage using the test outbox tool.
The tool is a real LangChain StructuredTool and is loaded through the normal
``get_available_tools`` + LangGraph ``ToolNode`` path. It is intentionally
only added to this test's AppConfig; production ``config.yaml`` does not
register it.
"""
from __future__ import annotations
import json
from types import SimpleNamespace
import pytest
import pytest_asyncio
from maf_test_helpers import make_maf_service
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from app.maf_roundtable.action_coordinator import ActionCoordinator
from app.maf_roundtable.tool_executor import ToolExecutor
from app.maf_roundtable.write_guard_bridge import (
MafWriteGuardInterceptor,
MafWriteGuardPolicy,
register_write_guard_service,
)
from deerflow.agents.middlewares.write_guard_middleware import WriteGuardMiddleware
from deerflow.config import get_app_config
from deerflow.config.tool_config import ToolConfig
from deerflow.config.write_guard_config import WriteGuardConfig, WriteGuardProviderConfig
from deerflow.persistence.maf_roundtable import make_maf_roundtable_store
from deerflow.persistence.maf_roundtable.model import Base
TEST_TOOL = "maf_test_outbox_write"
@pytest_asyncio.fixture
async def store():
engine = create_async_engine("sqlite+aiosqlite://", echo=False)
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
sf = async_sessionmaker(engine, expire_on_commit=False)
result = make_maf_roundtable_store(sf)
assert result is not None
yield result
await engine.dispose()
@pytest.fixture
def svc(store):
result = make_maf_service(store, enabled=True, admin_diagnostic_access=True)
register_write_guard_service(result)
return result
def _app_config():
config = get_app_config().model_copy(deep=True)
config.tools = [
*config.tools,
ToolConfig(
name=TEST_TOOL,
group="maf-e2e",
use="deerflow.tools.builtins.maf_test_outbox:maf_test_outbox_write",
),
]
config.write_guard = WriteGuardConfig(
enabled=True,
provider=WriteGuardProviderConfig(
policy_use="app.maf_roundtable.write_guard_bridge:make_policy",
interceptor_use="app.maf_roundtable.write_guard_bridge:make_interceptor",
),
)
return config
def _snapshot():
tool = {"name": TEST_TOOL, "side_effect": "write", "risk_level": "R2"}
return {
"tools": [tool],
"agents": [{
"agent_id": "seat-1",
"allowed_tool_names": [TEST_TOOL],
"excluded_tools": [],
"tools": [tool],
}],
}
async def _make_pending_action(svc, store):
session = await svc.create_session(user_id="u1", body={"task_text": "test"})
run = await svc.create_run(session_id=session["id"], user_id="u1")
coordinator = ActionCoordinator(svc, user_id="u1")
actions = await coordinator.intercept_write_tool_calls(
run_id=run["id"],
session_id=session["id"],
agent_id="seat-1",
turn_index=1,
tool_calls=[{
"name": TEST_TOOL,
"id": "call-outbox-1",
"args": {"operation_id": "op-1", "value": "controlled-value"},
}],
capability_snapshot=_snapshot(),
write_policy="confirm_all_writes",
)
assert len(actions) == 1
return session, run, coordinator, actions[0]
@pytest.mark.asyncio
async def test_real_outbox_requires_approval_and_executes_once(monkeypatch, tmp_path, svc, store):
"""Pending action has no side effect; approved ToolNode call writes once."""
monkeypatch.setenv("MAF_E2E_OUTBOX_DIR", str(tmp_path))
monkeypatch.setattr(
"deerflow.config.agents_config.load_agent_config",
lambda _agent_id: SimpleNamespace(tool_groups=["maf-e2e"], model=None),
)
session, run, coordinator, action = await _make_pending_action(svc, store)
outbox_file = tmp_path / "op-1.jsonl"
assert not outbox_file.exists(), "approval interception must not execute the tool"
assert await store.actions.mark_approved(action["id"]) is True
class Adapter:
class Client:
async def execute_tool_directly(self, **kwargs):
from deerflow.tools.approved_tool_executor import execute_approved_tool
return await execute_approved_tool(
app_config=_app_config(),
checkpointer=None,
store=None,
thread_id=kwargs["thread_id"],
user_id="u1",
session_id=kwargs["principal"].session_id,
run_id=kwargs["principal"].run_id,
agent_id=kwargs["agent_id"],
turn_index=kwargs["turn_index"],
model_name=None,
excluded_tools=[],
capability_snapshot=kwargs["capability_snapshot"],
write_policy=kwargs["write_policy"],
tool_name=kwargs["tool_name"],
params=kwargs["params"],
approved_action_id=kwargs["approved_action_id"],
action_record=kwargs["action_record"],
)
def __init__(self):
self._client = self.Client()
executor = ToolExecutor(
adapter=Adapter(),
action_coordinator=coordinator,
user_id="u1",
)
first = await executor.execute(
action_id=action["id"],
session_id=session["id"],
run_id=run["id"],
agent_id="seat-1",
thread_id="thread-1",
approved_tool=TEST_TOOL,
capability_snapshot=_snapshot(),
write_policy="confirm_all_writes",
)
second = await executor.execute(
action_id=action["id"],
session_id=session["id"],
run_id=run["id"],
agent_id="seat-1",
thread_id="thread-1",
approved_tool=TEST_TOOL,
capability_snapshot=_snapshot(),
write_policy="confirm_all_writes",
)
assert first["status"] == "succeeded"
assert second["status"] == "succeeded"
assert outbox_file.read_text(encoding="utf-8").count("\n") == 1
stored = await store.actions.get_action(action["id"])
assert stored["status"] == "succeeded"
assert stored["result"]["tool_result"]["external_object_id"] == "op-1"
@pytest.mark.asyncio
async def test_rejected_action_never_reaches_real_tool(monkeypatch, tmp_path, svc, store):
monkeypatch.setenv("MAF_E2E_OUTBOX_DIR", str(tmp_path))
_session, _run, _coordinator, action = await _make_pending_action(svc, store)
assert await store.actions.mark_rejected(action["id"]) is True
assert not list(tmp_path.iterdir())
assert (await store.actions.get_action(action["id"]))["status"] == "rejected"
@pytest.mark.asyncio
async def test_missing_secret_fails_closed_before_action_creation(monkeypatch, tmp_path, svc, store):
monkeypatch.delenv("MAF_ACTION_SECRET_KEY", raising=False)
monkeypatch.setenv("MAF_E2E_OUTBOX_DIR", str(tmp_path))
session = await svc.create_session(user_id="u1", body={"task_text": "test"})
run = await svc.create_run(session_id=session["id"], user_id="u1")
middleware = WriteGuardMiddleware(
policy=MafWriteGuardPolicy(),
interceptor=MafWriteGuardInterceptor(),
)
called = 0
context = {
"maf_run_id": run["id"],
"maf_session_id": session["id"],
"maf_user_id": "u1",
"agent_id": "seat-1",
"turn_index": 1,
"write_policy": "confirm_all_writes",
"capability_snapshot": _snapshot(),
}
request = SimpleNamespace(
tool_call={
"name": TEST_TOOL,
"id": "call-missing-key",
"args": {"operation_id": "op-missing-key", "value": "must-not-run"},
},
runtime=SimpleNamespace(context=context),
)
async def handler(_request):
nonlocal called
called += 1
return {"executed": True}
result = await middleware.awrap_tool_call(request, handler)
envelope = json.loads(result.content)
assert envelope["status"] == "intercept_failed"
assert called == 0
assert not list(tmp_path.iterdir())
assert await store.actions.list_actions(run_id=run["id"]) == []
@@ -0,0 +1,223 @@
"""Real HTTP approval → dispatcher → runner → ToolNode/outbox acceptance test."""
from __future__ import annotations
from types import SimpleNamespace
import pytest
from fastapi import FastAPI, Request
from httpx import ASGITransport, AsyncClient
from maf_test_helpers import make_maf_service
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from app.gateway.routers.maf_roundtable import router as maf_router
from app.maf_roundtable.action_coordinator import ActionCoordinator
from app.maf_roundtable.dispatcher import get_dispatcher
from app.maf_roundtable.runtime_client import DeerFlowRuntimeClient
from app.maf_roundtable.workflow_executors import RoundtableState
from app.maf_roundtable.write_guard_bridge import register_write_guard_service
from deerflow.config import get_app_config
from deerflow.config.maf_roundtable_config import MafRoundtableConfig
from deerflow.config.tool_config import ToolConfig
from deerflow.config.write_guard_config import WriteGuardConfig, WriteGuardProviderConfig
from deerflow.persistence.base import Base
from deerflow.persistence.maf_roundtable import make_maf_roundtable_store
from deerflow.tools.approved_tool_executor import execute_approved_tool
TEST_TOOL = "maf_test_outbox_write"
class _User:
def __init__(self, user_id: str) -> None:
self.id = user_id
self.system_role = "user"
def _snapshot() -> dict:
tool = {
"name": TEST_TOOL,
"side_effect": "write",
"risk_level": "R2",
"target_key_fields": ["operation_id"],
"target_key_namespace": "maf-e2e",
}
return {
"tools": [tool],
"agents": [{
"agent_id": "seat-1",
"allowed_tool_names": [TEST_TOOL],
"excluded_tools": [],
"tools": [tool],
}],
}
def _app_config():
config = get_app_config().model_copy(deep=True)
config.tools = [
*config.tools,
ToolConfig(
name=TEST_TOOL,
group="maf-e2e",
use="deerflow.tools.builtins.maf_test_outbox:maf_test_outbox_write",
),
]
config.write_guard = WriteGuardConfig(
enabled=True,
provider=WriteGuardProviderConfig(
policy_use="app.maf_roundtable.write_guard_bridge:make_policy",
interceptor_use="app.maf_roundtable.write_guard_bridge:make_interceptor",
),
)
return config
@pytest.mark.asyncio
async def test_http_approval_dispatches_real_tool_once(monkeypatch, tmp_path):
"""A real route decision reaches the ToolNode exactly once, even on replay."""
monkeypatch.setenv("MAF_ACTION_SECRET_KEY", "test-only-http-e2e-secret")
monkeypatch.setenv("MAF_E2E_OUTBOX_DIR", str(tmp_path))
monkeypatch.setattr(
"deerflow.config.agents_config.load_agent_config",
lambda _agent_id: SimpleNamespace(tool_groups=["maf-e2e"], model=None),
)
engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'maf_http_e2e.db'}")
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
sf = async_sessionmaker(engine, expire_on_commit=False)
store = make_maf_roundtable_store(sf)
assert store is not None
svc = make_maf_service(
store,
enabled=True,
config=MafRoundtableConfig(enabled=True, approval_timeout_seconds=300),
)
register_write_guard_service(svc)
app = FastAPI()
app.state.maf_roundtable_service = svc
app.include_router(maf_router)
@app.middleware("http")
async def test_auth(request: Request, call_next):
request.state.user = _User("u1")
return await call_next(request)
async def execute_directly(_client, **kwargs):
return await execute_approved_tool(
app_config=_app_config(),
checkpointer=None,
store=None,
thread_id=kwargs["thread_id"],
user_id=kwargs["principal"].user_id,
session_id=kwargs["principal"].session_id,
run_id=kwargs["principal"].run_id,
agent_id=kwargs["agent_id"],
turn_index=kwargs["turn_index"],
model_name=None,
excluded_tools=[],
capability_snapshot=kwargs["capability_snapshot"],
write_policy=kwargs["write_policy"],
tool_name=kwargs["tool_name"],
params=kwargs["params"],
approved_action_id=kwargs["approved_action_id"],
action_record=kwargs["action_record"],
)
async def no_model_finalizer(_adapter, state):
state.final_result = {"content": "http e2e final report"}
monkeypatch.setattr(DeerFlowRuntimeClient, "execute_tool_directly", execute_directly)
monkeypatch.setattr("app.maf_roundtable.workflow_runner.generate_final_report", no_model_finalizer)
session = await svc.create_session(user_id="u1", body={"task_text": "HTTP E2E"})
run = await svc.create_run(session_id=session["id"], user_id="u1")
coordinator = ActionCoordinator(svc, user_id="u1")
actions = await coordinator.intercept_write_tool_calls(
run_id=run["id"],
session_id=session["id"],
agent_id="seat-1",
turn_index=1,
tool_calls=[{
"name": TEST_TOOL,
"id": "http-outbox-call",
"args": {"operation_id": "http-op-1", "value": "controlled"},
}],
capability_snapshot=_snapshot(),
write_policy="confirm_all_writes",
)
action = actions[0]
state = RoundtableState(
session_id=session["id"],
run_id=run["id"],
task_text="HTTP E2E",
mode="structured",
model_name=None,
reasoning_effort=None,
max_rounds=1,
write_policy="confirm_all_writes",
manager_agent_id=None,
seat_agent_ids=["seat-1"],
thread_map={"seat-1": "thread-1"},
capability_snapshot=_snapshot(),
)
state.current_round = 1
state.status = "waiting_approval"
state.pending_approvals = [{
"action_id": action["id"],
"approval_id": action["approval_id"],
"agent_id": "seat-1",
"tool_name": TEST_TOOL,
}]
await store.runs.update_run(run["id"], {"status": "waiting_approval", "maf_checkpoint": state.to_dict()})
await store.sessions.update_session(
session["id"], user_id="u1", fields={"status": "waiting_approval"}
)
transport = ASGITransport(app=app)
async with AsyncClient(transport=transport, base_url="http://test") as client:
listed = await client.get(f"/api/maf-roundtable/runs/{run['id']}/actions")
assert listed.status_code == 200
listed_action = listed.json()["actions"][0]
assert "target_key" not in listed_action
assert "idempotency_key" not in listed_action
fetched = await client.get(f"/api/maf-roundtable/actions/{action['id']}")
assert fetched.status_code == 200
fetched_action = fetched.json()
assert "target_key" not in fetched_action
assert "idempotency_key" not in fetched_action
# The lease arbiter still needs the private key in persistence.
stored_before_decision = await store.actions.get_action(action["id"])
assert stored_before_decision["target_key"]
assert stored_before_decision["idempotency_key"]
response = await client.post(
f"/api/maf-roundtable/approvals/{action['approval_id']}/decision",
json={"decision": "approved"},
)
assert response.status_code == 200
assert response.json()["status"] == "approved"
duplicate = await client.post(
f"/api/maf-roundtable/approvals/{action['approval_id']}/decision",
json={"decision": "approved"},
)
assert duplicate.status_code == 409
outbox = tmp_path / "http-op-1.jsonl"
assert outbox.read_text(encoding="utf-8").count("\n") == 1
stored = await store.actions.get_action(action["id"])
assert stored["status"] == "succeeded"
assert stored["attempt_count"] == 1
assert (await store.runs.get_run(run["id"]))["status"] == "completed"
# A process/retry replay after the terminal transition returns the stored
# run and never reaches ToolNode again.
replay = await get_dispatcher(app).resume(run["id"], user_id="u1")
assert replay is not None and replay["status"] == "completed"
assert outbox.read_text(encoding="utf-8").count("\n") == 1
await engine.dispose()
@@ -1,7 +1,7 @@
"""User-isolation + table-isolation tests for the MAF roundtable experiment (MAF-01-07).
These tests focus on the security boundary: a non-owner cannot read or mutate
another user's session / run / action / approval, and the six MAF tables have
another user's session / run / action / lease / approval, and the nine MAF tables have
no foreign-key relationship to the legacy ``roundtable_*`` tables (the
experiment is fully isolated —see implementation plan §2.3 "不改原有的工程定义).
"""
@@ -17,6 +17,7 @@ from app.maf_roundtable.service import MafNotFoundError
from deerflow.persistence.base import Base
from deerflow.persistence.maf_roundtable import make_maf_roundtable_store
from deerflow.persistence.maf_roundtable.model import (
MafRoundtableActionLeaseRow,
MafRoundtableActionRow,
MafRoundtableActionSecretRow,
MafRoundtableApprovalRow,
@@ -149,7 +150,7 @@ async def test_admin_diagnostic_access_off_keeps_owner_scope(store):
@pytest.mark.asyncio
async def test_maf_tables_isolated_from_legacy_roundtable(store):
"""The six MAF tables must exist and none of their FKs may target a legacy
"""The nine MAF tables must exist and none of their FKs may target a legacy
roundtable_* table (MAF-01-02 / 退出标净 MAF 表和旀roundtable 表完全无关联外键)."""
_, engine = store
async with engine.connect() as conn:
@@ -162,6 +163,7 @@ async def test_maf_tables_isolated_from_legacy_roundtable(store):
"maf_roundtable_runs",
"maf_roundtable_events",
"maf_roundtable_actions",
"maf_roundtable_action_leases",
"maf_roundtable_approvals",
"maf_roundtable_seat_runs",
"maf_roundtable_action_secrets",
@@ -174,6 +176,7 @@ async def test_maf_tables_isolated_from_legacy_roundtable(store):
MafRoundtableRunRow,
MafRoundtableEventRow,
MafRoundtableActionRow,
MafRoundtableActionLeaseRow,
MafRoundtableActionSecretRow,
MafRoundtableApprovalRow,
MafRoundtableSeatRunRow,
@@ -770,12 +770,9 @@ def test_validate_maf_prerequisites_passes_when_all_met(monkeypatch):
reset_write_guard_config()
def test_validate_maf_prerequisites_requires_secret_key(monkeypatch):
"""validate_maf_prerequisites raises when MAF_ACTION_SECRET_KEY is missing."""
from app.maf_roundtable.service import (
MafConfigurationError,
validate_maf_prerequisites,
)
def test_validate_maf_prerequisites_allows_read_only_without_secret(monkeypatch):
"""A missing key leaves reads available; writes fail closed at interception."""
from app.maf_roundtable.service import validate_maf_prerequisites
from deerflow.config.write_guard_config import (
load_write_guard_config_from_dict,
reset_write_guard_config,
@@ -790,8 +787,34 @@ def test_validate_maf_prerequisites_requires_secret_key(monkeypatch):
"interceptor_use": "app.maf_roundtable.write_guard_bridge:make_interceptor",
},
})
with pytest.raises(MafConfigurationError):
validate_maf_prerequisites()
validate_maf_prerequisites()
finally:
reset_write_guard_config()
def test_write_guard_health_reports_missing_secret(monkeypatch):
from app.maf_roundtable.service import get_maf_write_guard_readiness
from deerflow.config.write_guard_config import (
load_write_guard_config_from_dict,
reset_write_guard_config,
)
monkeypatch.delenv("MAF_ACTION_SECRET_KEY", raising=False)
try:
load_write_guard_config_from_dict({
"enabled": True,
"provider": {
"policy_use": "app.maf_roundtable.write_guard_bridge:make_policy",
"interceptor_use": "app.maf_roundtable.write_guard_bridge:make_interceptor",
},
})
readiness = get_maf_write_guard_readiness()
assert readiness == {
"enabled": True,
"ready": False,
"secret_configured": False,
"status": "missing_secret",
}
finally:
reset_write_guard_config()