feat(maf): Phase 5 - action coordinator, approval flow, frontend MAF roundtable pages
- Backend: action_coordinator, capability_snapshot, event_normalizer, tool_executor updates - Backend: approved_tool_executor with write guard, action leases, migration script - Frontend: maf-roundtable module (API client, event timeline, approval cards, fact flow panel) - Frontend: MafRoutes, MafSessionsPage, MafRunDetailPage - Tests: action_leases, migration_phase5, phase5_entry_e2e, real_write_e2e/http_e2e - Docs: update implementation plan Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
21c4c898e7
commit
f82363d81f
55 files changed
+6304
-167
No files matched your search
@@ -6,8 +6,9 @@ import sys
|
||||
import threading
|
||||
from collections.abc import AsyncGenerator
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import datetime
|
||||
from time import perf_counter
|
||||
from datetime import datetime
|
||||
from time import perf_counter
|
||||
from typing import Any
|
||||
|
||||
from dotenv import load_dotenv
|
||||
from fastapi import FastAPI
|
||||
@@ -958,8 +959,63 @@ def _warn_if_multi_worker() -> None:
|
||||
pass
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
async def _scan_maf_approval_expiry(app: Any) -> list[str]:
|
||||
"""Expire due MAF approvals once, then resume affected workflows.
|
||||
|
||||
The store owns the conditional state transition and critical audit event.
|
||||
Only the worker that wins that CAS receives an affected run, so repeated
|
||||
scans cannot resume the same expiry twice.
|
||||
"""
|
||||
|
||||
service = getattr(app.state, "maf_roundtable_service", None)
|
||||
if service is None or not getattr(service, "enabled", False):
|
||||
return []
|
||||
|
||||
expired = await service.store.expire_due_approvals()
|
||||
run_ids = list(dict.fromkeys(
|
||||
str(item["run_id"])
|
||||
for item in expired
|
||||
if item.get("run_id")
|
||||
))
|
||||
if not run_ids:
|
||||
return []
|
||||
|
||||
from app.maf_roundtable.dispatcher import get_dispatcher
|
||||
|
||||
dispatcher = get_dispatcher(app)
|
||||
resumed: list[str] = []
|
||||
for run_id in run_ids:
|
||||
run = await service.store.runs.get_run(run_id)
|
||||
if run is None:
|
||||
continue
|
||||
session = await service.store.sessions.get_session(run["session_id"], user_id=None)
|
||||
if session is None:
|
||||
continue
|
||||
# Dispatcher.resume is a no-op for a terminal/non-waiting run; keeping
|
||||
# this call centralized preserves the restart and run-level CAS path.
|
||||
await dispatcher.resume(run_id, user_id=str(session.get("user_id") or "default"))
|
||||
resumed.append(run_id)
|
||||
return resumed
|
||||
|
||||
|
||||
async def _maf_approval_scanner_loop(app: Any) -> None:
|
||||
"""Continuously fail closed expired MAF approvals until shutdown."""
|
||||
|
||||
service = getattr(app.state, "maf_roundtable_service", None)
|
||||
config = getattr(service, "config", None)
|
||||
interval = max(1, int(getattr(config, "approval_scan_interval_seconds", 10) or 10))
|
||||
while True:
|
||||
try:
|
||||
await _scan_maf_approval_expiry(app)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception("MAF approval expiry scan failed (non-fatal)")
|
||||
await asyncio.sleep(interval)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
"""Application lifespan handler."""
|
||||
|
||||
# ?????? worker ?? ?? ?? AI ?? / ? agent ???????
|
||||
@@ -1123,12 +1179,27 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
|
||||
# Concurrency (system-pressure) sampler — lightweight, singleton-locked,
|
||||
# runtime-toggleable via system_settings.concurrency_monitor.enabled.
|
||||
try:
|
||||
app.state.concurrency_sampler_task = asyncio.create_task(_concurrency_sampler_loop(app))
|
||||
except Exception:
|
||||
logger.exception("Concurrency sampler failed to start (non-fatal)")
|
||||
|
||||
# Start IM channel service if any channels are configured
|
||||
try:
|
||||
app.state.concurrency_sampler_task = asyncio.create_task(_concurrency_sampler_loop(app))
|
||||
except Exception:
|
||||
logger.exception("Concurrency sampler failed to start (non-fatal)")
|
||||
|
||||
# MAF-only approval expiry scanner. Legacy DeerFlow, old roundtable
|
||||
# and AI-writing paths never enter this loop.
|
||||
try:
|
||||
maf_service = getattr(app.state, "maf_roundtable_service", None)
|
||||
if maf_service is not None and getattr(maf_service, "enabled", False):
|
||||
app.state.maf_approval_scanner_task = asyncio.create_task(
|
||||
_maf_approval_scanner_loop(app)
|
||||
)
|
||||
logger.info(
|
||||
"MAF approval expiry scanner started (interval=%ss)",
|
||||
getattr(maf_service.config, "approval_scan_interval_seconds", 10),
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("MAF approval expiry scanner failed to start (non-fatal)")
|
||||
|
||||
# Start IM channel service if any channels are configured
|
||||
try:
|
||||
from app.channels.service import start_channel_service
|
||||
|
||||
@@ -1187,16 +1258,26 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
logger.exception("Leaderboard snapshot scheduler shutdown error")
|
||||
|
||||
concurrency_sampler_task = getattr(app.state, "concurrency_sampler_task", None)
|
||||
if concurrency_sampler_task is not None:
|
||||
concurrency_sampler_task.cancel()
|
||||
try:
|
||||
await concurrency_sampler_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception:
|
||||
logger.exception("Concurrency sampler shutdown error")
|
||||
|
||||
# ?? 7 ????AI ?? graph ???? checkpointer / in-flight task?
|
||||
if concurrency_sampler_task is not None:
|
||||
concurrency_sampler_task.cancel()
|
||||
try:
|
||||
await concurrency_sampler_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception:
|
||||
logger.exception("Concurrency sampler shutdown error")
|
||||
|
||||
maf_approval_scanner_task = getattr(app.state, "maf_approval_scanner_task", None)
|
||||
if maf_approval_scanner_task is not None:
|
||||
maf_approval_scanner_task.cancel()
|
||||
try:
|
||||
await maf_approval_scanner_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception:
|
||||
logger.exception("MAF approval expiry scanner shutdown error")
|
||||
|
||||
# ?? 7 ????AI ?? graph ???? checkpointer / in-flight task?
|
||||
# thread ??? LangGraph Server ? checkpointer ???lifespan ???
|
||||
# ``langgraph_runtime()`` ? contextmanager ???????????????
|
||||
|
||||
|
||||
@@ -112,18 +112,25 @@ async def get_health(request: Request) -> HealthResponse:
|
||||
"""Feature-flag probe. Open read — the frontend reads ``enabled`` to
|
||||
decide whether to show the MAF entry. Does not require the store.
|
||||
"""
|
||||
from app.maf_roundtable.service import get_maf_write_guard_readiness
|
||||
from deerflow.config import get_app_config
|
||||
|
||||
cfg = get_app_config().maf_roundtable
|
||||
app_config = get_app_config()
|
||||
cfg = app_config.maf_roundtable
|
||||
svc = _get_service(request)
|
||||
store_available = svc is not None and svc.store is not None
|
||||
readiness = get_maf_write_guard_readiness(app_config=app_config)
|
||||
return HealthResponse(
|
||||
enabled=cfg.enabled,
|
||||
enabled=bool(cfg.enabled and svc is not None and svc.enabled),
|
||||
store_available=store_available,
|
||||
default_mode=cfg.default_mode,
|
||||
default_write_policy=cfg.default_write_policy,
|
||||
default_max_rounds=cfg.default_max_rounds,
|
||||
admin_diagnostic_access=cfg.admin_diagnostic_access,
|
||||
write_guard_enabled=readiness["enabled"],
|
||||
write_guard_ready=readiness["ready"],
|
||||
action_secret_configured=readiness["secret_configured"],
|
||||
write_guard_status=readiness["status"],
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -359,6 +359,50 @@ _CONTEXT_CONFIGURABLE_KEYS: frozenset[str] = frozenset(
|
||||
}
|
||||
)
|
||||
|
||||
# These fields activate or authorize MAF WriteGuard behavior. They may only
|
||||
# cross the Gateway boundary on the typed, server-created request from
|
||||
# ``DeerFlowRuntimeClient``. In particular, accepting them from a public
|
||||
# ``body.config`` would let a caller forge an active MAF run or a post-approval
|
||||
# passthrough.
|
||||
_MAF_PRIVATE_CONTEXT_KEYS: frozenset[str] = frozenset(
|
||||
{
|
||||
"maf_run_id",
|
||||
"maf_session_id",
|
||||
"maf_user_id",
|
||||
"turn_index",
|
||||
"write_policy",
|
||||
"capability_snapshot",
|
||||
"maf_approved_tool",
|
||||
"maf_approved_action_id",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _is_internal_maf_request(request: Any) -> bool:
|
||||
state = getattr(request, "state", None)
|
||||
return bool(getattr(state, "maf_internal", False))
|
||||
|
||||
|
||||
def _merge_internal_maf_context(config: dict[str, Any], context: Mapping[str, Any], request: Any) -> None:
|
||||
"""Forward server-generated MAF context, never public request fields."""
|
||||
if not _is_internal_maf_request(request):
|
||||
for container_name in ("context", "configurable"):
|
||||
container = config.get(container_name)
|
||||
if isinstance(container, dict):
|
||||
for key in _MAF_PRIVATE_CONTEXT_KEYS:
|
||||
container.pop(key, None)
|
||||
return
|
||||
|
||||
configurable = config.setdefault("configurable", {})
|
||||
runtime_context = config.setdefault("context", {})
|
||||
for key in _MAF_PRIVATE_CONTEXT_KEYS:
|
||||
if key not in context:
|
||||
continue
|
||||
if isinstance(configurable, dict):
|
||||
configurable.setdefault(key, context[key])
|
||||
if isinstance(runtime_context, dict):
|
||||
runtime_context.setdefault(key, context[key])
|
||||
|
||||
|
||||
def merge_run_context_overrides(config: dict[str, Any], context: Mapping[str, Any] | None) -> None:
|
||||
"""Merge whitelisted keys from ``body.context`` into both ``config['configurable']``
|
||||
@@ -1022,6 +1066,7 @@ async def start_run(
|
||||
if value is not None and top_level_key not in body_context:
|
||||
body_context[top_level_key] = value
|
||||
merge_run_context_overrides(config, body_context or None)
|
||||
_merge_internal_maf_context(config, body_context, request)
|
||||
|
||||
stream_modes = normalize_stream_modes(body.stream_mode)
|
||||
|
||||
|
||||
@@ -54,8 +54,12 @@ from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from math import isfinite
|
||||
from typing import Any
|
||||
|
||||
from app.maf_roundtable.sanitize import sanitize, sanitize_payload
|
||||
@@ -140,6 +144,90 @@ def tool_needs_approval(
|
||||
return risk_val >= 3
|
||||
|
||||
|
||||
def derive_target_key(
|
||||
tool_name: str,
|
||||
raw_args: Any,
|
||||
capability_snapshot: dict[str, Any] | None,
|
||||
agent_id: str | None = None,
|
||||
) -> str:
|
||||
"""Return a service-derived, opaque target lease key for a write action.
|
||||
|
||||
The model and browser never get a target-key input. Only the frozen,
|
||||
server-built capability metadata chooses the parameter fields used to
|
||||
identify a business object. Only stable scalar values are accepted;
|
||||
structured or otherwise unstable values conservatively collapse to the
|
||||
unidentified target. The canonical value is protected with a
|
||||
domain-separated HMAC keyed by ``MAF_ACTION_SECRET_KEY`` before it is
|
||||
persisted, so a database reader cannot brute-force ordinary object IDs
|
||||
with a public hash.
|
||||
|
||||
An unclassified/missing identifier deliberately shares one conservative
|
||||
key across writes (rather than guessing that different requests are safe
|
||||
to parallelize). Such tools are already write/R3 by default and require
|
||||
normal approval under the default policy.
|
||||
"""
|
||||
metadata: dict[str, Any] | None = None
|
||||
snapshot_tools = (capability_snapshot or {}).get("tools")
|
||||
if isinstance(snapshot_tools, list):
|
||||
metadata = next(
|
||||
(item for item in snapshot_tools if isinstance(item, dict) and item.get("name") == tool_name),
|
||||
None,
|
||||
)
|
||||
if metadata is None and agent_id:
|
||||
for agent in (capability_snapshot or {}).get("agents") or []:
|
||||
if not isinstance(agent, dict) or agent.get("agent_id") != agent_id:
|
||||
continue
|
||||
for item in agent.get("tools") or []:
|
||||
if isinstance(item, dict) and item.get("name") == tool_name:
|
||||
metadata = item
|
||||
break
|
||||
if metadata is not None:
|
||||
break
|
||||
|
||||
fields = metadata.get("target_key_fields") if isinstance(metadata, dict) else None
|
||||
namespace = metadata.get("target_key_namespace") if isinstance(metadata, dict) else None
|
||||
if not isinstance(fields, list) or not fields or not isinstance(raw_args, dict):
|
||||
canonical = {"scope": "unidentified"}
|
||||
else:
|
||||
valid_fields = [
|
||||
field for field in fields
|
||||
if isinstance(field, str) and field != "target_key" and field in raw_args
|
||||
]
|
||||
if len(valid_fields) != len(fields) or any(
|
||||
not _is_stable_scalar(raw_args[field]) for field in valid_fields
|
||||
):
|
||||
canonical = {"scope": "unidentified"}
|
||||
else:
|
||||
canonical = {
|
||||
"scope": "identified",
|
||||
"tool": tool_name,
|
||||
"namespace": namespace if isinstance(namespace, str) and namespace else "default",
|
||||
"target": {field: raw_args[field] for field in sorted(valid_fields)},
|
||||
}
|
||||
|
||||
encoded = json.dumps(canonical, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||
raw_secret = os.getenv("MAF_ACTION_SECRET_KEY", "")
|
||||
if not raw_secret.strip():
|
||||
raise MafActionSecretKeyError(
|
||||
"MAF_ACTION_SECRET_KEY is required before deriving a real-write target key"
|
||||
)
|
||||
domain = "maf-target-key:v1"
|
||||
digest = hmac.new(
|
||||
raw_secret.encode("utf-8"),
|
||||
domain.encode("ascii") + b"\x00" + encoded.encode("utf-8"),
|
||||
hashlib.sha256,
|
||||
).hexdigest()
|
||||
return f"maf-target:v1:{digest}"
|
||||
|
||||
|
||||
def _is_stable_scalar(value: Any) -> bool:
|
||||
"""Whether a configured target field can safely identify one target."""
|
||||
|
||||
if value is None or isinstance(value, (str, bool, int)):
|
||||
return True
|
||||
return isinstance(value, float) and isfinite(value)
|
||||
|
||||
|
||||
# ── Encryption ─────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -230,6 +318,10 @@ class ActionCoordinator:
|
||||
self._store = service.store
|
||||
self._user_id = user_id
|
||||
|
||||
def _approval_expires_at(self) -> datetime:
|
||||
seconds = int(getattr(self._service.config, "approval_timeout_seconds", 3600) or 3600)
|
||||
return datetime.now(UTC) + timedelta(seconds=seconds)
|
||||
|
||||
# ── Post-turn: capture intercepted write tool calls ──────────────────
|
||||
|
||||
async def intercept_write_tool_calls(
|
||||
@@ -268,6 +360,7 @@ class ActionCoordinator:
|
||||
|
||||
_side_effect, risk_level = classify_tool(tool_name, capability_snapshot, agent_id)
|
||||
idempotency_key = f"{run_id}:{agent_id}:{turn_index}:{tool_call_id or tool_name}"
|
||||
target_key = derive_target_key(tool_name, raw_args, capability_snapshot, agent_id)
|
||||
|
||||
# Sanitize args for audit visibility (never expose raw args).
|
||||
sanitized_args = sanitize(raw_args)
|
||||
@@ -290,7 +383,7 @@ class ActionCoordinator:
|
||||
logical_action_id=tool_call_id or None,
|
||||
idempotency_key=idempotency_key,
|
||||
risk_level=risk_level,
|
||||
target_key=None,
|
||||
target_key=target_key,
|
||||
turn_index=turn_index,
|
||||
arguments_sanitized=sanitized_args,
|
||||
arguments_encrypted=encrypted_args,
|
||||
@@ -303,6 +396,7 @@ class ActionCoordinator:
|
||||
},
|
||||
audit_event_agent_id=agent_id,
|
||||
audit_event_turn_index=turn_index,
|
||||
approval_expires_at=self._approval_expires_at(),
|
||||
)
|
||||
|
||||
created.append(action)
|
||||
@@ -322,7 +416,10 @@ class ActionCoordinator:
|
||||
approved. This guarantees at-most-once execution under concurrent
|
||||
``resume()`` / retries / restart.
|
||||
"""
|
||||
return await self._store.actions.claim_for_execution(action_id)
|
||||
lease_seconds = int(getattr(self._service.config, "target_lease_seconds", 900) or 900)
|
||||
return await self._store.claim_action_for_execution(
|
||||
action_id, lease_seconds=lease_seconds
|
||||
)
|
||||
|
||||
async def record_result(
|
||||
self,
|
||||
@@ -344,7 +441,7 @@ class ActionCoordinator:
|
||||
is a no-op — it must NOT overwrite the real result.
|
||||
"""
|
||||
sanitized_result = sanitize_payload(result) if result else None
|
||||
await self._store.actions.record_terminal_result(
|
||||
await self._store.record_action_terminal_result(
|
||||
action_id,
|
||||
status=status,
|
||||
result=sanitized_result,
|
||||
|
||||
@@ -66,7 +66,7 @@ def _seat_tools_for_agent(
|
||||
session_model_name: str | None,
|
||||
session_reasoning_effort: str | None,
|
||||
requested_excluded: list[str],
|
||||
) -> tuple[list[str], list[str], list[dict[str, str]]]:
|
||||
) -> tuple[list[str], list[str], list[dict[str, Any]]]:
|
||||
"""Resolve allowed names, effective excluded, and tool metadata for one seat."""
|
||||
agent_config = load_agent_config(agent_id)
|
||||
tool_groups = getattr(agent_config, "tool_groups", None) if agent_config else None
|
||||
@@ -88,10 +88,19 @@ def _seat_tools_for_agent(
|
||||
effective_excluded = sorted(excluded_set & allowed_set)
|
||||
effective_allowed = sorted(allowed_set - set(effective_excluded))
|
||||
|
||||
tools_meta: list[dict[str, str]] = []
|
||||
tools_meta: list[dict[str, Any]] = []
|
||||
maf_tool_metadata = getattr(
|
||||
getattr(app_config, "maf_roundtable", None), "tool_metadata", None
|
||||
) or {}
|
||||
for name in effective_allowed:
|
||||
extra = _config_extra_for_tool(app_config, name)
|
||||
tools_meta.append(resolve_tool_metadata(name, config_extra=extra))
|
||||
tools_meta.append(
|
||||
resolve_tool_metadata(
|
||||
name,
|
||||
config_extra=extra,
|
||||
maf_metadata=maf_tool_metadata,
|
||||
)
|
||||
)
|
||||
|
||||
return effective_allowed, effective_excluded, tools_meta
|
||||
|
||||
@@ -108,7 +117,7 @@ async def build_trusted_capability_snapshot(
|
||||
) -> dict[str, Any]:
|
||||
"""Build the frozen snapshot stored on the session and used by WriteGuard."""
|
||||
agents_out: list[dict[str, Any]] = []
|
||||
flat_tools: dict[str, dict[str, str]] = {}
|
||||
flat_tools: dict[str, dict[str, Any]] = {}
|
||||
|
||||
for p in participants:
|
||||
agent_id = str(p.get("agent_id") or "")
|
||||
|
||||
@@ -137,6 +137,16 @@ def make_action_rejected(agent_id: str, turn_index: int, *, action_id: str, appr
|
||||
return _make("action.rejected", agent_id, turn_index, {"action_id": action_id, "approval_id": approval_id, "decided_by": decided_by})
|
||||
|
||||
|
||||
def make_action_unknown(agent_id: str, turn_index: int, *, action_id: str, reason_code: str) -> dict[str, Any]:
|
||||
"""Record an inconclusive external outcome without leaking tool arguments."""
|
||||
return _make("action.unknown", agent_id, turn_index, {"action_id": action_id, "reason_code": reason_code})
|
||||
|
||||
|
||||
def make_action_waiting_target_lease(agent_id: str, turn_index: int, *, action_id: str, tool_name: str) -> dict[str, Any]:
|
||||
"""A second approved write is serialized behind an active target lease."""
|
||||
return _make("action.waiting_target_lease", agent_id, turn_index, {"action_id": action_id, "tool_name": tool_name})
|
||||
|
||||
|
||||
# ── workflow checkpoint events ─────────────────────────────────────────────
|
||||
|
||||
|
||||
|
||||
@@ -78,6 +78,10 @@ class _InternalState:
|
||||
def __init__(self, principal: ExecutionPrincipal) -> None:
|
||||
self.user = _PrincipalUser(principal)
|
||||
self.auth = None
|
||||
# This marker is set only on the server-created request used by the
|
||||
# MAF dispatcher. Gateway code uses it to distinguish trusted MAF
|
||||
# execution context from user-supplied run configuration.
|
||||
self.maf_internal = True
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -193,8 +193,6 @@ class ActionResponse(BaseModel):
|
||||
agent_id: str | None = None
|
||||
tool_name: str
|
||||
logical_action_id: str | None = None
|
||||
idempotency_key: str
|
||||
target_key: str | None = None
|
||||
risk_level: str
|
||||
status: str
|
||||
arguments: dict[str, Any] | None = None
|
||||
@@ -255,3 +253,7 @@ class HealthResponse(BaseModel):
|
||||
default_write_policy: str
|
||||
default_max_rounds: int
|
||||
admin_diagnostic_access: bool
|
||||
write_guard_enabled: bool
|
||||
write_guard_ready: bool
|
||||
action_secret_configured: bool
|
||||
write_guard_status: str
|
||||
@@ -79,6 +79,56 @@ class MafConfigurationError(RuntimeError):
|
||||
"""
|
||||
|
||||
|
||||
def get_maf_write_guard_readiness(*, app_config: Any | None = None) -> dict[str, Any]:
|
||||
"""Return non-sensitive WriteGuard readiness information for MAF health.
|
||||
|
||||
A missing secret intentionally does not disable read-only MAF work:
|
||||
WriteGuard fails closed before any external write can execute. ``ready``
|
||||
is therefore the signal that the complete approval + encryption path is
|
||||
usable, while ``enabled`` describes whether the middleware is mounted.
|
||||
"""
|
||||
import os
|
||||
|
||||
from deerflow.config.write_guard_config import get_write_guard_config
|
||||
from deerflow.reflection import resolve_variable
|
||||
|
||||
wg = getattr(app_config, "write_guard", None) if app_config is not None else None
|
||||
if wg is None:
|
||||
wg = get_write_guard_config()
|
||||
|
||||
provider = getattr(wg, "provider", None)
|
||||
provider_configured = bool(
|
||||
provider
|
||||
and getattr(provider, "policy_use", "")
|
||||
and getattr(provider, "interceptor_use", "")
|
||||
)
|
||||
bridge_resolvable = False
|
||||
if provider_configured:
|
||||
try:
|
||||
resolve_variable(provider.policy_use)
|
||||
resolve_variable(provider.interceptor_use)
|
||||
bridge_resolvable = True
|
||||
except Exception: # noqa: BLE001 - health must remain safe and non-fatal
|
||||
bridge_resolvable = False
|
||||
|
||||
secret_configured = bool((os.getenv("MAF_ACTION_SECRET_KEY") or "").strip())
|
||||
if not getattr(wg, "enabled", False):
|
||||
status = "disabled"
|
||||
elif not provider_configured or not bridge_resolvable:
|
||||
status = "provider_unavailable"
|
||||
elif not secret_configured:
|
||||
status = "missing_secret"
|
||||
else:
|
||||
status = "ready"
|
||||
|
||||
return {
|
||||
"enabled": bool(getattr(wg, "enabled", False)),
|
||||
"ready": status == "ready",
|
||||
"secret_configured": secret_configured,
|
||||
"status": status,
|
||||
}
|
||||
|
||||
|
||||
def validate_maf_prerequisites() -> None:
|
||||
"""Fail loudly if MAF is enabled without its full safety stack in place.
|
||||
|
||||
@@ -86,14 +136,13 @@ def validate_maf_prerequisites() -> None:
|
||||
1. ``write_guard.enabled`` is true (the approval guard is mounted);
|
||||
2. ``write_guard.provider`` is complete (``policy_use`` + ``interceptor_use``);
|
||||
3. the bridge factories actually resolve (the app-layer guard is importable);
|
||||
4. ``MAF_ACTION_SECRET_KEY`` is set (action params are encrypted with it).
|
||||
4. secret readiness is reported separately; a missing secret must not
|
||||
disable read-only MAF work, while writes fail closed before execution.
|
||||
|
||||
Raises ``MafConfigurationError`` on the first unmet condition. Callers on
|
||||
the startup path treat that as "force the feature off"; direct unit-test
|
||||
construction of :class:`MafRoundtableService` does NOT call this.
|
||||
"""
|
||||
import os
|
||||
|
||||
from deerflow.config.write_guard_config import get_write_guard_config
|
||||
from deerflow.reflection import resolve_variable
|
||||
|
||||
@@ -116,10 +165,6 @@ def validate_maf_prerequisites() -> None:
|
||||
raise MafConfigurationError(
|
||||
f"write_guard bridge factories are not resolvable: {exc}"
|
||||
) from exc
|
||||
if not (os.getenv("MAF_ACTION_SECRET_KEY") or "").strip():
|
||||
raise MafConfigurationError(
|
||||
"maf_roundtable.enabled=true requires the MAF_ACTION_SECRET_KEY env var"
|
||||
)
|
||||
|
||||
|
||||
# Session statuses that may be edited (config changes).
|
||||
@@ -150,24 +195,10 @@ class MafRoundtableService:
|
||||
# defaults when no config is injected (keeps unit tests lightweight).
|
||||
self._config = config or MafRoundtableConfig()
|
||||
|
||||
# Startup validation: if the MAF feature is enabled, the
|
||||
# ``MAF_ACTION_SECRET_KEY`` env var MUST be set — external-write
|
||||
# action params are encrypted with it and there is no default key.
|
||||
# A missing key forces the feature off (write paths then return 503)
|
||||
# rather than running with a publicly-known key. This never aborts
|
||||
# Gateway startup (the feature is experimental / opt-in).
|
||||
if enabled:
|
||||
import os
|
||||
|
||||
key = os.getenv("MAF_ACTION_SECRET_KEY", "")
|
||||
if not key or not key.strip():
|
||||
logger.error(
|
||||
"[maf-roundtable] MAF_ACTION_SECRET_KEY is not set — "
|
||||
"disabling the feature (write paths will return 503). "
|
||||
"Set a 32+ char random string before enabling "
|
||||
"maf_roundtable.enabled."
|
||||
)
|
||||
enabled = False
|
||||
# A missing MAF_ACTION_SECRET_KEY does not disable read-only MAF
|
||||
# operation. The action coordinator raises before creating or
|
||||
# executing an external-write action, so writes fail closed until the
|
||||
# deployment injects the secret through the environment.
|
||||
self._enabled = enabled
|
||||
|
||||
@property
|
||||
@@ -539,10 +570,17 @@ class MafRoundtableService:
|
||||
async def list_actions(
|
||||
self, *, run_id: str, user_id: str | None, is_admin: bool = False
|
||||
) -> list[dict[str, Any]]:
|
||||
from deerflow.persistence.maf_roundtable.actions import action_to_public_dict
|
||||
|
||||
run = await self.get_run(run_id=run_id, user_id=user_id, is_admin=is_admin)
|
||||
return await self._store.actions.list_actions(run_id=run["id"])
|
||||
return [
|
||||
action_to_public_dict(action)
|
||||
for action in await self._store.actions.list_actions(run_id=run["id"])
|
||||
]
|
||||
|
||||
async def get_action(self, *, action_id: str, user_id: str | None, is_admin: bool = False) -> dict[str, Any]:
|
||||
from deerflow.persistence.maf_roundtable.actions import action_to_public_dict
|
||||
|
||||
action = await self._store.actions.get_action(action_id)
|
||||
if action is None:
|
||||
raise MafNotFoundError("action", action_id)
|
||||
@@ -550,7 +588,7 @@ class MafRoundtableService:
|
||||
session = await self._store.sessions.get_session(action["session_id"], user_id=scope)
|
||||
if session is None:
|
||||
raise MafNotFoundError("action", action_id)
|
||||
return action
|
||||
return action_to_public_dict(action)
|
||||
|
||||
# ── approvals ──────────────────────────────────────────────────────────
|
||||
|
||||
@@ -558,6 +596,8 @@ class MafRoundtableService:
|
||||
self, *, run_id: str, user_id: str | None, is_admin: bool = False
|
||||
) -> list[dict[str, Any]]:
|
||||
run = await self.get_run(run_id=run_id, user_id=user_id, is_admin=is_admin)
|
||||
# Reads are observational. Expiry is performed by the dedicated
|
||||
# scanner; decision/resume paths retain the fail-closed pre-CAS check.
|
||||
return await self._store.approvals.list_approvals(run_id=run["id"])
|
||||
|
||||
async def decide_approval(
|
||||
@@ -579,6 +619,11 @@ class MafRoundtableService:
|
||||
if is_admin and await self._store.sessions.get_session(approval["session_id"], user_id=None) is not None:
|
||||
raise MafForbiddenError("approval", approval_id)
|
||||
raise MafNotFoundError("approval", approval_id)
|
||||
# The timeout transition is server-owned and happens before the
|
||||
# decision CAS, so an expired approval can never be revived by a late
|
||||
# browser request.
|
||||
if approval.get("run_id"):
|
||||
await self._store.expire_due_approvals(run_id=approval["run_id"])
|
||||
try:
|
||||
updated = await self._store.approvals.decide(approval_id, decision=decision, decided_by=user_id)
|
||||
except ValueError as exc:
|
||||
|
||||
@@ -113,6 +113,15 @@ class ToolExecutor:
|
||||
)
|
||||
return {"status": "failed", "error": f"action state machine error: {exc}"}
|
||||
|
||||
if claim == "target_busy":
|
||||
logger.info("ToolExecutor: target lease is held for action %s", action_id)
|
||||
return {"status": "waiting_target_lease", "error": "target lease is held by another action"}
|
||||
if claim == "target_reconciliation_required":
|
||||
logger.warning("ToolExecutor: expired target lease requires reconciliation for %s", action_id)
|
||||
return {
|
||||
"status": "unknown",
|
||||
"error": "target lease expired during another action; reconciliation required",
|
||||
}
|
||||
if claim != "claimed":
|
||||
existing = await self._coordinator._store.actions.get_action(action_id) # noqa: SLF001
|
||||
if claim == "in_progress":
|
||||
@@ -128,9 +137,8 @@ class ToolExecutor:
|
||||
return {"status": claim, "action": existing}
|
||||
|
||||
if not approved_tool:
|
||||
await self._coordinator.record_result(
|
||||
action_id, status="failed",
|
||||
result={"error": "no approved tool name — cannot execute"},
|
||||
await self._record_result_or_log(
|
||||
action_id, status="failed", result={"error": "no approved tool name — cannot execute"}
|
||||
)
|
||||
return {"status": "failed", "error": "no approved tool name"}
|
||||
|
||||
@@ -147,11 +155,14 @@ class ToolExecutor:
|
||||
"ToolExecutor: action %s params unavailable (%s): %s",
|
||||
action_id, exc.reason, safe_msg,
|
||||
)
|
||||
await self._coordinator.record_result(
|
||||
persisted = await self._record_result_or_log(
|
||||
action_id, status="failed",
|
||||
result={"error": f"action secret unavailable: {exc.reason}"},
|
||||
)
|
||||
return {"status": "failed", "error": safe_msg}
|
||||
return {
|
||||
"status": "failed" if persisted else "unknown",
|
||||
"error": safe_msg if persisted else "action result persistence failed",
|
||||
}
|
||||
|
||||
# 3. Read the stored action record (authoritative turn_index + the
|
||||
# run/session/agent/tool the harness will verify against).
|
||||
@@ -191,34 +202,83 @@ class ToolExecutor:
|
||||
"ToolExecutor: direct tool call '%s' raised: %s",
|
||||
approved_tool, safe_msg,
|
||||
)
|
||||
await self._coordinator.record_result(
|
||||
action_id, status="failed", result={"error": safe_msg},
|
||||
# An exception can arrive after the remote request was accepted
|
||||
# (network disconnect / timeout), so fail closed as ``unknown``.
|
||||
persisted = await self._record_result_or_log(
|
||||
action_id, status="unknown", result={"error": "tool_transport_exception"}
|
||||
)
|
||||
return {"status": "failed", "error": safe_msg}
|
||||
return {
|
||||
"status": "unknown",
|
||||
"error": safe_msg if persisted else "action result persistence failed",
|
||||
}
|
||||
|
||||
# 5. Record terminal result (state-machine constrained: only
|
||||
# running → terminal; a lost race is a no-op).
|
||||
outcome_status = outcome.get("status")
|
||||
if outcome_status == "succeeded":
|
||||
await self._coordinator.record_result(
|
||||
action_id, status="succeeded",
|
||||
result={"tool_result": outcome.get("result")},
|
||||
tool_result = outcome.get("result")
|
||||
external_object_id = (
|
||||
str(tool_result.get("external_object_id"))
|
||||
if isinstance(tool_result, dict)
|
||||
and tool_result.get("external_object_id")
|
||||
else None
|
||||
)
|
||||
return {"status": "succeeded", "tool_result": outcome.get("result")}
|
||||
persisted = await self._record_result_or_log(
|
||||
action_id, status="succeeded",
|
||||
result={"tool_result": tool_result},
|
||||
external_object_id=external_object_id,
|
||||
)
|
||||
if not persisted:
|
||||
# The external side effect succeeded but its local terminal
|
||||
# record did not. Never retry; the retained running lease is
|
||||
# later reconciled to unknown by recovery.
|
||||
return {"status": "unknown", "error": "action result persistence failed"}
|
||||
return {"status": "succeeded", "tool_result": tool_result}
|
||||
elif outcome_status == "unknown":
|
||||
# Inconclusive external result (timeout / disconnect / gateway):
|
||||
# record ``unknown`` — never auto-retry, a human must reconcile.
|
||||
error = outcome.get("error", "inconclusive external result")
|
||||
await self._coordinator.record_result(
|
||||
persisted = await self._record_result_or_log(
|
||||
action_id, status="unknown", result={"error": error},
|
||||
)
|
||||
return {"status": "unknown", "error": error}
|
||||
return {"status": "unknown", "error": error if persisted else "action result persistence failed"}
|
||||
else:
|
||||
error = outcome.get("error", "tool execution failed")
|
||||
await self._coordinator.record_result(
|
||||
persisted = await self._record_result_or_log(
|
||||
action_id, status="failed", result={"error": error},
|
||||
)
|
||||
return {"status": "failed", "error": error}
|
||||
return {
|
||||
"status": "failed" if persisted else "unknown",
|
||||
"error": error if persisted else "action result persistence failed",
|
||||
}
|
||||
|
||||
async def _record_result_or_log(
|
||||
self,
|
||||
action_id: str,
|
||||
*,
|
||||
status: str,
|
||||
result: dict[str, Any],
|
||||
external_object_id: str | None = None,
|
||||
) -> bool:
|
||||
"""Persist a terminal outcome without opening a retry path after I/O.
|
||||
|
||||
If this write fails after a real tool reports success, returning an
|
||||
``unknown`` outcome lets the parent run stop for reconciliation while
|
||||
the still-held lease prevents a replay of the external call.
|
||||
"""
|
||||
try:
|
||||
await self._coordinator.record_result(
|
||||
action_id,
|
||||
status=status,
|
||||
result=result,
|
||||
external_object_id=external_object_id,
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
logger.exception(
|
||||
"ToolExecutor: could not persist terminal result for action %s", action_id
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
async def record_unknown(self, action_id: str, *, reason: str) -> None:
|
||||
"""Record an ``unknown`` outcome (network timeout / inconclusive).
|
||||
@@ -230,6 +290,6 @@ class ToolExecutor:
|
||||
"ToolExecutor: action %s outcome unknown — %s (no auto-retry)",
|
||||
action_id, reason,
|
||||
)
|
||||
await self._coordinator.record_result(
|
||||
action_id, status="unknown", result={"reason": reason},
|
||||
await self._record_result_or_log(
|
||||
action_id, status="unknown", result={"reason": reason}
|
||||
)
|
||||
@@ -316,6 +316,10 @@ class WorkflowRunner:
|
||||
logger.warning("Cannot resume run %s: no pending_approvals in checkpoint", run_id)
|
||||
return run
|
||||
|
||||
# Expire due approvals before inspecting the checkpoint. This is a
|
||||
# fail-closed server transition; a late HTTP approval cannot revive it.
|
||||
await self._service.store.expire_due_approvals(run_id=run_id)
|
||||
|
||||
# Check if any approval is still pending — if so, don't resume yet.
|
||||
any_pending = False
|
||||
for pa in state.pending_approvals:
|
||||
@@ -365,14 +369,15 @@ class WorkflowRunner:
|
||||
from app.maf_roundtable.event_normalizer import make_action_rejected
|
||||
# Conditional transition (waiting_approval → rejected) — never
|
||||
# overwrite a terminal state via a generic update.
|
||||
await self._service.store.actions.mark_rejected(action_id)
|
||||
await self._adapter._emit_critical(run_id, make_action_rejected(
|
||||
pa.get("agent_id", ""),
|
||||
state.current_round,
|
||||
action_id=action_id,
|
||||
approval_id=approval_id,
|
||||
decided_by=approval.get("decided_by", ""),
|
||||
))
|
||||
changed = await self._service.store.actions.mark_rejected(action_id)
|
||||
if changed:
|
||||
await self._adapter._emit_critical(run_id, make_action_rejected(
|
||||
pa.get("agent_id", ""),
|
||||
state.current_round,
|
||||
action_id=action_id,
|
||||
approval_id=approval_id,
|
||||
decided_by=approval.get("decided_by", ""),
|
||||
))
|
||||
action_outcomes.append({"action_id": action_id, "status": "rejected"})
|
||||
# Phase-3 P1-3: move the ledger row out of pending into the
|
||||
# failed/unknown partition as rejected.
|
||||
@@ -478,6 +483,37 @@ class WorkflowRunner:
|
||||
# P0-3: determine parent run status from action outcomes.
|
||||
statuses = {o.get("status") for o in action_outcomes}
|
||||
|
||||
# A target held by another approved write remains pending for an
|
||||
# explicit later resume. It is never retried automatically.
|
||||
if "waiting_target_lease" in statuses:
|
||||
from app.maf_roundtable.event_normalizer import make_action_waiting_target_lease
|
||||
|
||||
state.status = "waiting_approval"
|
||||
for outcome in action_outcomes:
|
||||
if outcome.get("status") == "waiting_target_lease":
|
||||
await self._adapter._emit_critical(
|
||||
run_id,
|
||||
make_action_waiting_target_lease(
|
||||
next(
|
||||
(str(pa.get("agent_id") or "") for pa in state.pending_approvals if pa.get("action_id") == outcome.get("action_id")),
|
||||
"",
|
||||
),
|
||||
state.current_round,
|
||||
action_id=str(outcome.get("action_id") or ""),
|
||||
tool_name=str(outcome.get("tool_name") or ""),
|
||||
),
|
||||
)
|
||||
await self._service.store.runs.update_run(run_id, {
|
||||
"status": "waiting_approval",
|
||||
"maf_checkpoint": state.to_dict(),
|
||||
})
|
||||
await self._service.store.sessions.update_session(
|
||||
session_id,
|
||||
user_id=self._user_id,
|
||||
fields={"status": "waiting_approval", "fact_ledger": state.fact_ledger},
|
||||
)
|
||||
return await self._service.store.runs.get_run(run_id)
|
||||
|
||||
# While any action is still in_progress (another executor running it),
|
||||
# the parent run stays running — never write finished_at, never finalize.
|
||||
if "in_progress" in statuses:
|
||||
@@ -572,6 +608,44 @@ class WorkflowRunner:
|
||||
if session is None:
|
||||
return run
|
||||
|
||||
expired_actions = await self._service.store.recover_expired_action_leases(run_id=run_id)
|
||||
if expired_actions:
|
||||
from app.maf_roundtable.event_normalizer import make_action_unknown
|
||||
|
||||
expired_ids = {action["id"] for action in expired_actions}
|
||||
for action in expired_actions:
|
||||
record_action_transition(
|
||||
state.fact_ledger,
|
||||
action_id=action["id"],
|
||||
tool_name=action.get("tool_name", ""),
|
||||
status="unknown",
|
||||
agent_id=action.get("agent_id", ""),
|
||||
)
|
||||
await self._adapter._emit_critical(
|
||||
run_id,
|
||||
make_action_unknown(
|
||||
action.get("agent_id", ""),
|
||||
int(action.get("turn_index") or state.current_round),
|
||||
action_id=action["id"],
|
||||
reason_code="target_lease_expired_during_execution",
|
||||
),
|
||||
)
|
||||
state.pending_approvals = [
|
||||
item for item in state.pending_approvals
|
||||
if item.get("action_id") not in expired_ids
|
||||
]
|
||||
state.status = "waiting_input"
|
||||
await self._service.store.runs.update_run(run_id, {
|
||||
"status": "waiting_input",
|
||||
"maf_checkpoint": state.to_dict(),
|
||||
})
|
||||
await self._service.store.sessions.update_session(
|
||||
state.session_id,
|
||||
user_id=self._user_id,
|
||||
fields={"status": "waiting_input", "fact_ledger": state.fact_ledger},
|
||||
)
|
||||
return await self._service.store.runs.get_run(run_id)
|
||||
|
||||
# Ensure threads still bound.
|
||||
participants = await self._service.store.participants.list_participants(state.session_id)
|
||||
thread_map = await self._adapter.ensure_threads(state.session_id, run_id, participants)
|
||||
|
||||
Reference in new issue
Block a user