fix(maf): harden stage-4 quality gate (P0/P1)
P0-1 migration: detect the same-named stage-3 execution_stage CHECK by sqltext (must contain rework + manager_review), SQLite rebuild / PG-MySQL drop+add, PG boolean default TRUE, validate_schema checks constraint content + seat_tasks status CHECK/unique/FKs; tests start from a real stage-3 schema. P0-2: a missing/invalid manager review (timeout/exception/empty/invalid JSON/missing per-seat decision) is never an acceptance - system rework under the cap, contract fallback (reassign/controlled_gap/ask_user) afterwards, source=system events. P0-3: unaccepted deliveries are isolated from the fact ledger, cross-review and final report (deferred merge, delivery_json envelope, accepted_task_results in checkpoint, accepted-only synthesis, no turn_results fallback). P0-4: seat-task init is atomic get-or-create over the full expected set; recovery fills missing tasks without overwriting frozen contracts; quality gate blocks missing required tasks. P1: zero config values honoured; duplicate detection persists SHA-256 conclusion digests in history (survives restarts); findings never satisfy evidence, evidence_refs must resolve to trusted ledger refs (unverified_evidence_ref); conversation SSE projection tests added and manager/finalizer messages are no longer mislabelled as seat.delivery.submitted.
This commit is contained in:
1 parent
a786cdcad0
commit
1aeee86984
24 files changed
+4706
-47
No files matched your search
@@ -32,14 +32,45 @@ from app.maf_roundtable.dispatch_contracts import (
|
||||
MafDispatchCatalog,
|
||||
MafDispatchPlan,
|
||||
)
|
||||
from app.maf_roundtable.event_normalizer import (
|
||||
make_manager_review_decision,
|
||||
make_seat_delivery_submitted,
|
||||
make_seat_reassigned,
|
||||
make_seat_rework_assigned,
|
||||
make_stage_quality_gate,
|
||||
)
|
||||
from app.maf_roundtable.fact_ledger import (
|
||||
confirm_candidates,
|
||||
empty_ledger,
|
||||
known_evidence_refs,
|
||||
merge_seat_output,
|
||||
normalize_ledger,
|
||||
record_action_transition,
|
||||
record_tool_result,
|
||||
)
|
||||
from app.maf_roundtable.quality_contracts import (
|
||||
OUTCOME_ACCEPTED,
|
||||
OUTCOME_ASK_USER,
|
||||
OUTCOME_CONTROLLED_GAP,
|
||||
OUTCOME_REASSIGNED,
|
||||
OUTCOME_REWORK,
|
||||
TASK_ACCEPTED,
|
||||
TASK_CONTROLLED_GAP,
|
||||
TASK_EXHAUSTED,
|
||||
TASK_REASSIGNED,
|
||||
TASK_REWORK,
|
||||
TASK_SUBMITTED,
|
||||
TASK_TERMINAL_STATUSES,
|
||||
MafTaskContract,
|
||||
build_manager_review_prompt,
|
||||
build_task_contract,
|
||||
check_delivery_deterministic,
|
||||
conclusion_digest,
|
||||
delivery_from_seat_output,
|
||||
finalize_review_decision,
|
||||
make_controlled_gap,
|
||||
parse_manager_review,
|
||||
)
|
||||
from app.maf_roundtable.seat_output import (
|
||||
build_repair_prompt,
|
||||
parse_cross_review_output,
|
||||
@@ -85,6 +116,7 @@ class RoundtableState:
|
||||
dispatch_plan: dict[str, Any] | None = None,
|
||||
candidate_seat_agent_ids: list[str] | None = None,
|
||||
dispatch_contract_enabled: bool | None = None,
|
||||
quality_gate_enabled: bool | None = None,
|
||||
) -> None:
|
||||
self.session_id = session_id
|
||||
self.run_id = run_id
|
||||
@@ -110,6 +142,14 @@ class RoundtableState:
|
||||
if dispatch_contract_enabled is None
|
||||
else bool(dispatch_contract_enabled)
|
||||
)
|
||||
# Stage 4 (R3-Q): quality gate for V2 structured dispatch. Defaults to
|
||||
# the dispatch-contract value when not explicitly resolved, so
|
||||
# checkpoints written before Stage 4 keep the new behavior.
|
||||
self.quality_gate_enabled = (
|
||||
bool(self.dispatch_contract_enabled)
|
||||
if quality_gate_enabled is None
|
||||
else bool(quality_gate_enabled)
|
||||
)
|
||||
plan_seats = list((dispatch_plan or {}).get("seat_agent_ids") or [])
|
||||
self.candidate_seat_agent_ids = list(
|
||||
(dispatch_plan or {}).get("candidate_seat_agent_ids")
|
||||
@@ -132,6 +172,19 @@ class RoundtableState:
|
||||
self.stage: str = "planning"
|
||||
self.partial_failure: bool = False
|
||||
self.cross_review_result: dict[str, Any] | None = None
|
||||
# Stage 4 (R3-Q): per-task quality state machine summary.
|
||||
# task_key → {status, agent_id, original_agent_id, attempt_count,
|
||||
# rework_count, reassign_count, required}
|
||||
self.seat_task_statuses: dict[str, Any] = {}
|
||||
# Controlled-gap records to be carried into the final report's risk
|
||||
# section. Each entry is a make_controlled_gap(...) dict.
|
||||
self.controlled_gaps: list[dict[str, Any]] = []
|
||||
# Stage 4 (R3-Q): the ONLY seat outputs that cross-review and the
|
||||
# final synthesis may consume when the quality gate is enabled. An
|
||||
# entry is appended exactly when a task's final decision is
|
||||
# ``accepted`` — reworked / reassigned / gap deliveries never land
|
||||
# here, so unaccepted content cannot leak into the final report.
|
||||
self.accepted_task_results: list[dict[str, Any]] = []
|
||||
|
||||
def mark_completed(self, node_id: str) -> None:
|
||||
if node_id not in self.completed_nodes:
|
||||
@@ -159,6 +212,7 @@ class RoundtableState:
|
||||
"intent_type": self.intent_type,
|
||||
"dispatch_plan": self.dispatch_plan,
|
||||
"dispatch_contract_enabled": self.dispatch_contract_enabled,
|
||||
"quality_gate_enabled": self.quality_gate_enabled,
|
||||
"candidate_seat_agent_ids": list(self.candidate_seat_agent_ids),
|
||||
"selected_seat_agent_ids": self.selected_seat_agent_ids,
|
||||
"current_round": self.current_round,
|
||||
@@ -173,6 +227,9 @@ class RoundtableState:
|
||||
"stage": self.stage,
|
||||
"partial_failure": self.partial_failure,
|
||||
"cross_review_result": self.cross_review_result,
|
||||
"seat_task_statuses": dict(self.seat_task_statuses),
|
||||
"controlled_gaps": list(self.controlled_gaps),
|
||||
"accepted_task_results": list(self.accepted_task_results),
|
||||
}
|
||||
|
||||
@classmethod
|
||||
@@ -196,6 +253,7 @@ class RoundtableState:
|
||||
dispatch_plan=d.get("dispatch_plan"),
|
||||
candidate_seat_agent_ids=d.get("candidate_seat_agent_ids"),
|
||||
dispatch_contract_enabled=d.get("dispatch_contract_enabled"),
|
||||
quality_gate_enabled=d.get("quality_gate_enabled"),
|
||||
)
|
||||
state.current_round = d.get("current_round", 0)
|
||||
state.turn_results = d.get("turn_results", [])
|
||||
@@ -212,6 +270,9 @@ class RoundtableState:
|
||||
state.stage = d.get("stage") or "planning"
|
||||
state.partial_failure = bool(d.get("partial_failure", False))
|
||||
state.cross_review_result = d.get("cross_review_result")
|
||||
state.seat_task_statuses = dict(d.get("seat_task_statuses") or {})
|
||||
state.controlled_gaps = list(d.get("controlled_gaps") or [])
|
||||
state.accepted_task_results = list(d.get("accepted_task_results") or [])
|
||||
state.candidate_seat_agent_ids = list(
|
||||
d.get("candidate_seat_agent_ids") or state.candidate_seat_agent_ids
|
||||
)
|
||||
@@ -863,12 +924,24 @@ class SeatExecutor(Executor):
|
||||
agent_id: str,
|
||||
*,
|
||||
action_coordinator: ActionCoordinator | None = None,
|
||||
execution_stage: str | None = None,
|
||||
defer_fact_ledger_merge: bool = False,
|
||||
) -> None:
|
||||
super().__init__(id=f"seat_{agent_id}")
|
||||
self._adapter = adapter
|
||||
self._state = state
|
||||
self._agent_id = agent_id
|
||||
self._action_coordinator = action_coordinator
|
||||
# Stage 4 (R3-Q): optional override for the execution_stage passed to
|
||||
# run_seat_turn. When None, falls back to the mode-based default
|
||||
# ("expert" for structured, "group_chat" for group_chat).
|
||||
self._execution_stage = execution_stage
|
||||
# Stage 4 (R3-Q): when True the structured output is validated, tool
|
||||
# results are audited and events are recorded as usual, but the output
|
||||
# is NOT merged into the fact ledger. The quality loop merges it only
|
||||
# if the manager's final decision accepts the delivery — reworked,
|
||||
# reassigned or gapped content must never become candidate facts.
|
||||
self._defer_fact_ledger_merge = defer_fact_ledger_merge
|
||||
|
||||
@handler(input=Any, output=dict)
|
||||
async def handle(self, message: Any, ctx: WorkflowContext) -> None:
|
||||
@@ -918,7 +991,7 @@ class SeatExecutor(Executor):
|
||||
run_id=self._state.run_id,
|
||||
agent_id=self._agent_id,
|
||||
turn_index=self._state.current_round,
|
||||
execution_stage="group_chat" if self._state.mode == "group_chat" else "expert",
|
||||
execution_stage=self._execution_stage or ("group_chat" if self._state.mode == "group_chat" else "expert"),
|
||||
message=self._state.task_text,
|
||||
thread_id=thread_id,
|
||||
model_name=caps.get("model_name") or self._state.model_name,
|
||||
@@ -956,7 +1029,7 @@ class SeatExecutor(Executor):
|
||||
schema_result = await self._validate_output(raw_content)
|
||||
content = schema_result.get("raw_content") or raw_content
|
||||
|
||||
if schema_result.get("output"):
|
||||
if schema_result.get("output") and not self._defer_fact_ledger_merge:
|
||||
merge_seat_output(self._state.fact_ledger, schema_result["output"], agent_id=self._agent_id)
|
||||
|
||||
await self._capture_pending_approvals(response.tool_calls or [])
|
||||
@@ -981,7 +1054,7 @@ class SeatExecutor(Executor):
|
||||
# Phase-3 P0-1: record which execution stage produced this turn so
|
||||
# recovery can distinguish an expert's original output from its
|
||||
# format_repair / cross_review / finalizer re-runs.
|
||||
"execution_stage": "group_chat" if self._state.mode == "group_chat" else "expert",
|
||||
"execution_stage": self._execution_stage or ("group_chat" if self._state.mode == "group_chat" else "expert"),
|
||||
"content": content,
|
||||
"tool_calls": response.tool_calls,
|
||||
"error": response.error,
|
||||
@@ -1160,8 +1233,21 @@ class CrossReviewExecutor(Executor):
|
||||
|
||||
self._state.stage = "review"
|
||||
|
||||
# Barrier: ensure every seat has an outcome (fan-in should guarantee this).
|
||||
review_seats = self._state.selected_seat_agent_ids or self._state.seat_agent_ids
|
||||
# P0-3: with the Stage-4 quality gate the cross-review may only see
|
||||
# ACCEPTED seat results — reworked/reassigned/gapped deliveries never
|
||||
# enter the review prompt (their content is already absent from the
|
||||
# fact ledger because the merge was deferred until acceptance).
|
||||
if self._state.quality_gate_enabled:
|
||||
accepted_results = list(self._state.accepted_task_results)
|
||||
review_seats = sorted({
|
||||
str(r.get("agent_id") or "")
|
||||
for r in accepted_results
|
||||
if r.get("agent_id")
|
||||
})
|
||||
else:
|
||||
accepted_results = None
|
||||
# Barrier: ensure every seat has an outcome (fan-in should guarantee this).
|
||||
review_seats = self._state.selected_seat_agent_ids or self._state.seat_agent_ids
|
||||
missing = [a for a in review_seats if a not in self._state.seat_outcomes]
|
||||
for aid in missing:
|
||||
self._state.seat_outcomes[aid] = {"status": "missing", "error": "no_outcome"}
|
||||
@@ -1193,9 +1279,16 @@ class CrossReviewExecutor(Executor):
|
||||
thread_id = self._state.thread_map.get(manager_id or "", "")
|
||||
if manager_id and thread_id:
|
||||
try:
|
||||
if accepted_results is not None:
|
||||
seat_results_for_review: list[dict[str, Any]] = accepted_results
|
||||
else:
|
||||
seat_results_for_review = [
|
||||
r for r in self._state.turn_results
|
||||
if r.get("agent_id") in review_seats
|
||||
]
|
||||
prompt = build_manager_synthesis_input(
|
||||
task_text=self._state.task_text,
|
||||
seat_results=[r for r in self._state.turn_results if r.get("agent_id") in review_seats],
|
||||
seat_results=seat_results_for_review,
|
||||
fact_ledger=self._state.fact_ledger,
|
||||
)
|
||||
# Surface the candidate ids + the evidence refs the ledger has
|
||||
@@ -1257,6 +1350,9 @@ class CrossReviewExecutor(Executor):
|
||||
or review.get("conflicts")
|
||||
or review.get("tool_failures")
|
||||
or review.get("schema_status") == "schema_invalid"
|
||||
# Stage 4: a controlled gap is authoritative — the cross-review
|
||||
# decision must reflect it even when everything else looks clean.
|
||||
or (self._state.quality_gate_enabled and self._state.controlled_gaps)
|
||||
)
|
||||
review["decision"] = "controlled_gap" if has_quality_gap else "accept"
|
||||
review["reason"] = (
|
||||
@@ -1549,16 +1645,24 @@ async def generate_final_report(adapter: Any, state: RoundtableState) -> dict[st
|
||||
manager_id = state.manager_agent_id or "coordinator"
|
||||
thread_id = state.thread_map.get(manager_id, "")
|
||||
|
||||
usable = [
|
||||
r for r in state.turn_results
|
||||
if not r.get("error") or r.get("content")
|
||||
]
|
||||
if state.partial_failure:
|
||||
usable = [r for r in usable if (state.seat_outcomes.get(r.get("agent_id")) or {}).get("status") == "completed"]
|
||||
if state.quality_gate_enabled:
|
||||
# P0-3: Stage 4 synthesizes ONLY accepted task results. There is
|
||||
# deliberately NO fallback to ``turn_results`` — when no delivery was
|
||||
# accepted, the report must be built from the controlled-gap risk
|
||||
# section alone instead of smuggling rejected content back in.
|
||||
seat_results_for_synthesis: list[dict[str, Any]] = list(state.accepted_task_results)
|
||||
else:
|
||||
usable = [
|
||||
r for r in state.turn_results
|
||||
if not r.get("error") or r.get("content")
|
||||
]
|
||||
if state.partial_failure:
|
||||
usable = [r for r in usable if (state.seat_outcomes.get(r.get("agent_id")) or {}).get("status") == "completed"]
|
||||
seat_results_for_synthesis = usable or state.turn_results
|
||||
|
||||
synthesis_input = build_manager_synthesis_input(
|
||||
task_text=state.task_text,
|
||||
seat_results=usable or state.turn_results,
|
||||
seat_results=seat_results_for_synthesis,
|
||||
fact_ledger=state.fact_ledger,
|
||||
)
|
||||
if state.partial_failure:
|
||||
@@ -1573,6 +1677,32 @@ async def generate_final_report(adapter: Any, state: RoundtableState) -> dict[st
|
||||
if state.cross_review_result:
|
||||
synthesis_input += f"\n\n【交叉审查摘要】\n{state.cross_review_result.get('summary', '')[:2000]}"
|
||||
|
||||
# Stage 4 (R3-Q): controlled gaps must reach the final report traceably and
|
||||
# must NEVER be presented as verified facts. They are injected into the
|
||||
# synthesis prompt as an explicit risk/pending-info section.
|
||||
if state.controlled_gaps:
|
||||
gap_lines = [
|
||||
f"- 任务 {gap.get('task_key')}(席位 {gap.get('agent_id')},"
|
||||
f"尝试 {gap.get('attempts', 0)} 次):{gap.get('reason')}"
|
||||
for gap in state.controlled_gaps
|
||||
]
|
||||
synthesis_input += (
|
||||
"\n\n【受控缺口(尚未获得合格席位交付,严禁写成已验证事实)】\n"
|
||||
+ "\n".join(gap_lines)
|
||||
+ "\n要求:把上述每一条缺口完整列入最终结论的『风险/待补信息』章节,"
|
||||
"逐条说明其对结论的影响;不得将缺口内容陈述为已验证结论。"
|
||||
)
|
||||
|
||||
has_gaps = bool(state.controlled_gaps)
|
||||
quality_decision = (
|
||||
"controlled_gap"
|
||||
if has_gaps
|
||||
else (
|
||||
(state.cross_review_result or {}).get("decision")
|
||||
or ("controlled_gap" if state.partial_failure else "accept")
|
||||
)
|
||||
)
|
||||
|
||||
if thread_id:
|
||||
response = await adapter.run_seat_turn(
|
||||
session_id=state.session_id,
|
||||
@@ -1595,11 +1725,9 @@ async def generate_final_report(adapter: Any, state: RoundtableState) -> dict[st
|
||||
"content": response.content,
|
||||
"agent_id": manager_id,
|
||||
"partial_failure": state.partial_failure,
|
||||
"degraded": state.partial_failure,
|
||||
"quality_decision": (
|
||||
(state.cross_review_result or {}).get("decision")
|
||||
or ("controlled_gap" if state.partial_failure else "accept")
|
||||
),
|
||||
"degraded": state.partial_failure or has_gaps,
|
||||
"quality_decision": quality_decision,
|
||||
"controlled_gaps": list(state.controlled_gaps),
|
||||
}
|
||||
else:
|
||||
state.final_result = {
|
||||
@@ -1607,11 +1735,9 @@ async def generate_final_report(adapter: Any, state: RoundtableState) -> dict[st
|
||||
"agent_id": manager_id,
|
||||
"note": "no_manager_thread",
|
||||
"partial_failure": state.partial_failure,
|
||||
"degraded": state.partial_failure,
|
||||
"quality_decision": (
|
||||
(state.cross_review_result or {}).get("decision")
|
||||
or ("controlled_gap" if state.partial_failure else "accept")
|
||||
),
|
||||
"degraded": state.partial_failure or has_gaps,
|
||||
"quality_decision": quality_decision,
|
||||
"controlled_gaps": list(state.controlled_gaps),
|
||||
}
|
||||
|
||||
await adapter._emit_critical(state.run_id, make_run_completed(manager_id, state.current_round + 1, {
|
||||
@@ -1659,6 +1785,617 @@ class WaitingTerminalExecutor(Executor):
|
||||
await ctx.yield_output({"status": "waiting_approval"})
|
||||
|
||||
|
||||
class _SilentWorkflowCtx:
|
||||
"""WorkflowContext stand-in for SeatExecutor invoked INSIDE the quality
|
||||
loop. The embedded SeatExecutor reuses the outer executor's ctx when
|
||||
called directly, which would otherwise publish its messages with the
|
||||
outer node's source id and re-enter the graph. A silent ctx discards
|
||||
those intermediate messages — the quality loop reads the delivery state
|
||||
from ``turn_results`` / ``seat_outcomes`` instead."""
|
||||
|
||||
async def send_message(self, msg: Any) -> None:
|
||||
return None
|
||||
|
||||
async def yield_output(self, msg: Any) -> None:
|
||||
return None
|
||||
|
||||
|
||||
class WaitingInputTerminalExecutor(Executor):
|
||||
"""Terminal executor for the Stage-4 ask_user / waiting_input path.
|
||||
|
||||
Reached when a seat task's fallback policy requests user clarification
|
||||
(``ask_user``) and the manager cannot resolve the gap. The run ends in
|
||||
``waiting_input`` — never as a disguised success.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__(id="waiting_input_terminal")
|
||||
|
||||
@handler(input=Any, workflow_output=dict)
|
||||
async def handle(self, message: Any, ctx: WorkflowContext) -> None:
|
||||
await ctx.yield_output({"status": "waiting_input"})
|
||||
|
||||
|
||||
class QualitySeatTasksExecutor(Executor):
|
||||
"""Stage 4 (R3-Q): run every seat task through the acceptance state machine.
|
||||
|
||||
Replaces the plain SeatExecutor fan-out for V2 structured dispatch. One
|
||||
executor drives all tasks in bounded quality rounds::
|
||||
|
||||
delivering → submitted → manager_review →
|
||||
accepted | rework(→delivering) | reassigned(→delivering) |
|
||||
controlled_gap | ask_user(→ waiting_input)
|
||||
|
||||
Every delivery runs a deterministic quality check first; the manager model
|
||||
then proposes one structured decision per task, and the server finalizes it
|
||||
via :func:`quality_contracts.finalize_review_decision` — a delivery that
|
||||
failed the deterministic checks can NEVER be accepted, reworks are capped
|
||||
by the frozen contract, and an exhausted task ends as ``controlled_gap``
|
||||
(or an explicit ``ask_user`` pause) instead of being disguised as success.
|
||||
|
||||
Task rows persist in ``maf_roundtable_seat_tasks``; reassignment only
|
||||
changes the task's ``agent_id`` (a server-validated backup candidate) and
|
||||
never creates a new task.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
adapter: Any,
|
||||
state: RoundtableState,
|
||||
*,
|
||||
action_coordinator: ActionCoordinator | None = None,
|
||||
max_rework_attempts: int = 2,
|
||||
max_reassignments: int = 1,
|
||||
min_delivery_chars: int = 40,
|
||||
min_delivery_evidence: int = 1,
|
||||
fallback_policy: str = "reassign_then_gap",
|
||||
) -> None:
|
||||
super().__init__(id="quality_seat_tasks")
|
||||
self._adapter = adapter
|
||||
self._state = state
|
||||
self._action_coordinator = action_coordinator
|
||||
self._max_rework_attempts = max_rework_attempts
|
||||
self._max_reassignments = max_reassignments
|
||||
self._min_delivery_chars = min_delivery_chars
|
||||
self._min_delivery_evidence = min_delivery_evidence
|
||||
self._fallback_policy = fallback_policy
|
||||
self._tasks: list[dict[str, Any]] = []
|
||||
|
||||
@property
|
||||
def _store(self) -> Any:
|
||||
return self._adapter._service.store
|
||||
|
||||
@handler(input=Any, output=dict)
|
||||
async def handle(self, message: Any, ctx: WorkflowContext) -> None:
|
||||
if self._state.is_completed("quality_seat_tasks"):
|
||||
await ctx.send_message({
|
||||
"tasks": dict(self._state.seat_task_statuses),
|
||||
"replayed": True,
|
||||
})
|
||||
return
|
||||
|
||||
self._state.stage = "experts"
|
||||
tasks = await self._load_or_create_tasks()
|
||||
self._tasks = tasks
|
||||
base_round = max(self._state.current_round, 1)
|
||||
# P1: duplicate detection is seeded from the persisted task history
|
||||
# (SHA-256 digests of prior conclusions), so a restart cannot re-submit
|
||||
# an already-rejected conclusion and pass it off as new content.
|
||||
prior_conclusions: dict[str, set[str]] = {}
|
||||
for task in tasks:
|
||||
digests = {
|
||||
str(h.get("conclusion_sha256"))
|
||||
for h in (task.get("history") or [])
|
||||
if isinstance(h, dict) and h.get("conclusion_sha256")
|
||||
}
|
||||
prior_conclusions[str(task.get("task_key") or "")] = digests
|
||||
|
||||
# Bounded quality loop: initial delivery + max_rework_attempts reworks
|
||||
# per task budget, plus one delivery round per possible reassignment.
|
||||
max_rounds = (self._max_rework_attempts + 1) * (self._max_reassignments + 1) + 1
|
||||
for round_offset in range(max_rounds):
|
||||
self._state.current_round = base_round + round_offset
|
||||
active = [t for t in tasks if t["status"] not in TASK_TERMINAL_STATUSES]
|
||||
if not active or self._state.status == "waiting_input":
|
||||
break
|
||||
|
||||
submitted: list[dict[str, Any]] = []
|
||||
for task in active:
|
||||
outcome = await self._deliver(task, prior_conclusions)
|
||||
if outcome is not None:
|
||||
submitted.append(outcome)
|
||||
|
||||
if not submitted:
|
||||
continue
|
||||
|
||||
reviews = await self._manager_review(submitted)
|
||||
for task in submitted:
|
||||
final_outcome = await self._apply_decision(task, reviews.get(task["agent_id"]))
|
||||
if final_outcome == OUTCOME_ASK_USER:
|
||||
self._state.status = "waiting_input"
|
||||
self._state.stage = "waiting_input"
|
||||
|
||||
self._state.seat_task_statuses = {
|
||||
t["task_key"]: {
|
||||
"task_key": t["task_key"],
|
||||
"status": t["status"],
|
||||
"agent_id": t["agent_id"],
|
||||
"original_agent_id": t["original_agent_id"],
|
||||
"attempt_count": t["attempt_count"],
|
||||
"rework_count": t["rework_count"],
|
||||
"reassign_count": t["reassign_count"],
|
||||
"required": bool(t["required"]),
|
||||
}
|
||||
for t in tasks
|
||||
}
|
||||
self._state.mark_completed("quality_seat_tasks")
|
||||
await ctx.send_message({"tasks": dict(self._state.seat_task_statuses)})
|
||||
|
||||
# ── task init (atomic, recovery-safe) ───────────────────────────────────
|
||||
|
||||
def _expected_task_specs(self) -> list[dict[str, Any]]:
|
||||
"""The server-computed expected task set from the dispatch assignments.
|
||||
|
||||
``task_key`` is stable per original assignment, so recovery always
|
||||
derives the identical key set — existing rows are kept, rows that a
|
||||
crashed initialization never created are filled back in.
|
||||
"""
|
||||
plan = self._state.dispatch_plan or {}
|
||||
assignments = plan.get("assignments") or []
|
||||
specs: list[dict[str, Any]] = []
|
||||
for index, assignment in enumerate(assignments, start=1):
|
||||
agent_id = str(assignment.get("agent_id") or "").strip()
|
||||
if not agent_id:
|
||||
continue
|
||||
contract = build_task_contract(
|
||||
task_goal=str(assignment.get("task_goal") or ""),
|
||||
delivery_requirements=list(assignment.get("delivery_requirements") or []),
|
||||
required=True,
|
||||
max_rework_attempts=self._max_rework_attempts,
|
||||
fallback_policy=self._fallback_policy, # type: ignore[arg-type]
|
||||
)
|
||||
specs.append({
|
||||
"task_key": f"{index:02d}:{agent_id}",
|
||||
"agent_id": agent_id,
|
||||
"contract": contract.model_dump(),
|
||||
"required": True,
|
||||
})
|
||||
return specs
|
||||
|
||||
async def _load_or_create_tasks(self) -> list[dict[str, Any]]:
|
||||
"""Atomic get-or-create over the FULL expected task set (P0-4).
|
||||
|
||||
Never short-circuits on "some tasks already exist": a crash mid-init
|
||||
must not permanently drop the remaining required seats. Existing rows
|
||||
keep their frozen contracts; concurrent recoveries converge without
|
||||
unique-key errors.
|
||||
"""
|
||||
return await self._store.seat_tasks.get_or_create_tasks(
|
||||
session_id=self._state.session_id,
|
||||
run_id=self._state.run_id,
|
||||
expected=self._expected_task_specs(),
|
||||
)
|
||||
|
||||
# ── delivery step ───────────────────────────────────────────────────────
|
||||
|
||||
async def _deliver(
|
||||
self,
|
||||
task: dict[str, Any],
|
||||
prior_conclusions: dict[str, set[str]],
|
||||
) -> dict[str, Any] | None:
|
||||
"""Run one delivery via SeatExecutor (approvals / ledger / outcomes
|
||||
reuse the existing seat path), then submit it with its deterministic
|
||||
check result. Returns the submitted task dict or ``None`` if the seat
|
||||
run hard-failed without any output."""
|
||||
agent_id = task["agent_id"]
|
||||
|
||||
# Re-delivery must not replay the completed node.
|
||||
node_id = f"seat_{agent_id}"
|
||||
if node_id in self._state.completed_nodes:
|
||||
self._state.completed_nodes.remove(node_id)
|
||||
|
||||
# Reassigned tasks: add the validated backup seat to the selected set
|
||||
# and ensure its participant row + thread before it delivers.
|
||||
if task["status"] == TASK_REASSIGNED:
|
||||
if self._state.selected_seat_agent_ids is not None and agent_id not in self._state.selected_seat_agent_ids:
|
||||
self._state.selected_seat_agent_ids.append(agent_id)
|
||||
await _ensure_selected_seat_threads(self._adapter, self._state)
|
||||
|
||||
is_rework = task["attempt_count"] > 0
|
||||
seat_exec = SeatExecutor(
|
||||
self._adapter,
|
||||
self._state,
|
||||
agent_id,
|
||||
action_coordinator=self._action_coordinator,
|
||||
execution_stage="rework" if is_rework else None,
|
||||
# P0-3: the delivery must not touch the fact ledger until the
|
||||
# manager's final decision accepts it.
|
||||
defer_fact_ledger_merge=True,
|
||||
)
|
||||
await seat_exec.handle({}, _SilentWorkflowCtx())
|
||||
|
||||
turn_row = next(
|
||||
(
|
||||
r for r in reversed(self._state.turn_results)
|
||||
if r.get("agent_id") == agent_id
|
||||
and r.get("turn_index") == self._state.current_round
|
||||
),
|
||||
None,
|
||||
)
|
||||
raw_content = str((turn_row or {}).get("content") or "")
|
||||
structured_output = (turn_row or {}).get("structured_output")
|
||||
schema_status = str((turn_row or {}).get("schema_status") or "ok")
|
||||
delivery = delivery_from_seat_output(
|
||||
structured_output=structured_output, raw_content=raw_content
|
||||
)
|
||||
prior = prior_conclusions.get(task["task_key"])
|
||||
if prior is None:
|
||||
prior = set()
|
||||
prior_conclusions[task["task_key"]] = prior
|
||||
# P1: evidence must be traceable to the fact ledger (recorded evidence
|
||||
# or successful tool results). Self-made findings / invented refs fail
|
||||
# the deterministic gate.
|
||||
verified_refs = known_evidence_refs(self._state.fact_ledger)
|
||||
check = check_delivery_deterministic(
|
||||
delivery=delivery,
|
||||
raw_content=raw_content,
|
||||
schema_status=schema_status,
|
||||
min_length=self._min_delivery_chars,
|
||||
min_evidence=self._min_delivery_evidence,
|
||||
prior_conclusion_digests=prior,
|
||||
verified_evidence_refs=verified_refs,
|
||||
)
|
||||
digest = conclusion_digest(delivery.conclusion) if delivery.conclusion else None
|
||||
if digest:
|
||||
prior.add(digest)
|
||||
|
||||
task["attempt_count"] = int(task.get("attempt_count") or 0) + 1
|
||||
history = list(task.get("history") or [])
|
||||
history.append({
|
||||
"attempt": task["attempt_count"],
|
||||
"agent_id": agent_id,
|
||||
"round": self._state.current_round,
|
||||
"schema_status": schema_status,
|
||||
"failure_codes": list(check["failure_codes"]),
|
||||
# P1: persist the digest only — never the conclusion body — so
|
||||
# duplicate detection survives restarts.
|
||||
"conclusion_sha256": digest,
|
||||
})
|
||||
# P0-3: persisted delivery envelope. ``contract_delivery`` is the
|
||||
# acceptance-facing view; ``validated_seat_output`` is the schema-
|
||||
# validated raw SeatTurnOutput kept back until acceptance merges it
|
||||
# into the fact ledger. Runtime objects / tool-call arguments are
|
||||
# never written here.
|
||||
task["delivery"] = {
|
||||
"contract_delivery": delivery.model_dump(),
|
||||
"validated_seat_output": structured_output
|
||||
if isinstance(structured_output, dict)
|
||||
else None,
|
||||
}
|
||||
task["deterministic_check"] = check
|
||||
task["history"] = history
|
||||
# In-memory only (never persisted): the turn row of this attempt, so
|
||||
# an acceptance can merge exactly what was submitted.
|
||||
task["_turn_row"] = turn_row
|
||||
await self._store.seat_tasks.update_task(task["id"], {
|
||||
"status": TASK_SUBMITTED,
|
||||
"attempt_count": task["attempt_count"],
|
||||
"delivery": task["delivery"],
|
||||
"deterministic_check": check,
|
||||
"history": history,
|
||||
})
|
||||
task["status"] = TASK_SUBMITTED
|
||||
|
||||
await self._adapter._emit_critical(self._state.run_id, make_seat_delivery_submitted(
|
||||
agent_id,
|
||||
self._state.current_round,
|
||||
task_key=task["task_key"],
|
||||
attempt=task["attempt_count"],
|
||||
deterministic_passed=bool(check["passed"]),
|
||||
failure_codes=list(check["failure_codes"]),
|
||||
))
|
||||
return task
|
||||
|
||||
# ── manager review step ─────────────────────────────────────────────────
|
||||
|
||||
async def _manager_review(self, tasks: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
"""One batched manager review call for the round's submissions.
|
||||
|
||||
Returns ``{agent_id: MafManagerReviewItem}``. Any failure (timeout,
|
||||
runtime error, empty output, invalid JSON, schema failure) degrades to
|
||||
an empty dict — every task then finalizes as an INVALID review
|
||||
(``source=system``): a system rework while the cap allows it, or the
|
||||
contract's exhaustion outcome afterwards. A delivery is never accepted
|
||||
without a valid manager review, and a seat missing from a partially
|
||||
valid review is treated the same way.
|
||||
"""
|
||||
manager_id = self._state.manager_agent_id
|
||||
thread_id = self._state.thread_map.get(manager_id or "", "")
|
||||
if not (manager_id and thread_id):
|
||||
return {}
|
||||
|
||||
review_items: list[dict[str, Any]] = []
|
||||
for task in tasks:
|
||||
contract = task.get("contract") or {}
|
||||
envelope = task.get("delivery") or {}
|
||||
# The manager reviews the contract-facing delivery view; the raw
|
||||
# validated seat output stays server-side until acceptance.
|
||||
contract_delivery = envelope.get("contract_delivery") or envelope
|
||||
review_items.append({
|
||||
"agent_id": task["agent_id"],
|
||||
"task_key": task["task_key"],
|
||||
"attempt": task["attempt_count"],
|
||||
"contract": {
|
||||
"objective": contract.get("objective"),
|
||||
"expected_deliverable": contract.get("expected_deliverable"),
|
||||
"acceptance_criteria": list(contract.get("acceptance_criteria") or []),
|
||||
"required": bool(contract.get("required", True)),
|
||||
},
|
||||
"delivery": contract_delivery,
|
||||
"check": task.get("deterministic_check") or {},
|
||||
})
|
||||
prompt = build_manager_review_prompt(
|
||||
task_text=self._state.task_text, review_items=review_items
|
||||
)
|
||||
try:
|
||||
response = await asyncio.wait_for(
|
||||
self._adapter.run_seat_turn(
|
||||
session_id=self._state.session_id,
|
||||
run_id=self._state.run_id,
|
||||
agent_id=manager_id,
|
||||
turn_index=self._state.current_round,
|
||||
execution_stage="manager_review",
|
||||
message=prompt,
|
||||
thread_id=thread_id,
|
||||
model_name=self._state.model_name,
|
||||
reasoning_effort=self._state.reasoning_effort,
|
||||
task_text=prompt,
|
||||
stage_instruction="返回严格的总控验收 JSON(每席位一条 decision)",
|
||||
fact_ledger=self._state.fact_ledger,
|
||||
capability_snapshot=self._state.capability_snapshot,
|
||||
write_policy=self._state.write_policy,
|
||||
wait_timeout_seconds=self._state.seat_timeout_seconds,
|
||||
),
|
||||
timeout=self._state.seat_timeout_seconds,
|
||||
)
|
||||
except Exception: # noqa: BLE001 - degraded to deterministic finalization
|
||||
return {}
|
||||
if response.error:
|
||||
return {}
|
||||
model, _errors = parse_manager_review(response.content or "")
|
||||
if model is None:
|
||||
return {}
|
||||
return {item.agent_id: item for item in model.decisions}
|
||||
|
||||
# ── decision application ────────────────────────────────────────────────
|
||||
|
||||
async def _apply_decision(
|
||||
self,
|
||||
task: dict[str, Any],
|
||||
review_item: Any | None,
|
||||
) -> str:
|
||||
"""Finalize and persist one task's review decision; returns the outcome."""
|
||||
contract = MafTaskContract.model_validate(task.get("contract") or {})
|
||||
check = task.get("deterministic_check") or {"passed": False, "failure_codes": []}
|
||||
final = finalize_review_decision(
|
||||
contract=contract,
|
||||
check_result=check,
|
||||
review_item=review_item,
|
||||
rework_count=int(task.get("rework_count") or 0),
|
||||
reassign_count=int(task.get("reassign_count") or 0),
|
||||
has_backup_seat=bool(self._backup_agents(task)),
|
||||
max_reassignments=self._max_reassignments,
|
||||
)
|
||||
outcome = final["decision"]
|
||||
|
||||
await self._adapter._emit_critical(self._state.run_id, make_manager_review_decision(
|
||||
task["agent_id"],
|
||||
self._state.current_round,
|
||||
task_key=task["task_key"],
|
||||
decision=outcome,
|
||||
reason=final["reason"],
|
||||
source=final["source"],
|
||||
attempt=task["attempt_count"],
|
||||
))
|
||||
|
||||
if outcome == OUTCOME_ACCEPTED:
|
||||
task["status"] = TASK_ACCEPTED
|
||||
# P0-3: only an accepted delivery may enter the fact ledger and
|
||||
# the accepted result set consumed by cross-review / the final
|
||||
# synthesis. Reworked / reassigned / gapped attempts never do.
|
||||
envelope = task.get("delivery") or {}
|
||||
validated = envelope.get("validated_seat_output")
|
||||
if isinstance(validated, dict):
|
||||
merge_seat_output(
|
||||
self._state.fact_ledger, validated, agent_id=task["agent_id"]
|
||||
)
|
||||
turn_row = task.get("_turn_row") or {}
|
||||
self._state.accepted_task_results.append({
|
||||
"task_key": task["task_key"],
|
||||
"agent_id": task["agent_id"],
|
||||
"attempt": task["attempt_count"],
|
||||
"accepted": True,
|
||||
"source": final["source"],
|
||||
"turn_index": turn_row.get("turn_index", self._state.current_round),
|
||||
"content": str(turn_row.get("content") or ""),
|
||||
"execution_stage": turn_row.get("execution_stage"),
|
||||
"schema_status": turn_row.get("schema_status"),
|
||||
"structured_output": validated if isinstance(validated, dict) else None,
|
||||
})
|
||||
await self._store.seat_tasks.update_task(task["id"], {
|
||||
"status": TASK_ACCEPTED,
|
||||
"review_decision": final,
|
||||
})
|
||||
|
||||
elif outcome == OUTCOME_REWORK:
|
||||
task["rework_count"] = int(task.get("rework_count") or 0) + 1
|
||||
task["status"] = TASK_REWORK
|
||||
await self._store.seat_tasks.update_task(task["id"], {
|
||||
"status": TASK_REWORK,
|
||||
"rework_count": task["rework_count"],
|
||||
"review_decision": final,
|
||||
})
|
||||
await self._adapter._emit_critical(self._state.run_id, make_seat_rework_assigned(
|
||||
task["agent_id"],
|
||||
self._state.current_round,
|
||||
task_key=task["task_key"],
|
||||
rework_count=task["rework_count"],
|
||||
requirements=list(final.get("revision_requirements") or []),
|
||||
))
|
||||
|
||||
elif outcome == OUTCOME_REASSIGNED:
|
||||
from_agent_id = task["agent_id"]
|
||||
backup = self._pick_backup(task)
|
||||
task["reassign_count"] = int(task.get("reassign_count") or 0) + 1
|
||||
task["agent_id"] = backup
|
||||
task["status"] = TASK_REASSIGNED
|
||||
await self._store.seat_tasks.update_task(task["id"], {
|
||||
"agent_id": backup,
|
||||
"status": TASK_REASSIGNED,
|
||||
"reassign_count": task["reassign_count"],
|
||||
"review_decision": final,
|
||||
})
|
||||
await self._adapter._emit_critical(self._state.run_id, make_seat_reassigned(
|
||||
backup,
|
||||
self._state.current_round,
|
||||
task_key=task["task_key"],
|
||||
from_agent_id=from_agent_id,
|
||||
reassign_count=task["reassign_count"],
|
||||
))
|
||||
|
||||
elif outcome == OUTCOME_CONTROLLED_GAP:
|
||||
gap = make_controlled_gap(
|
||||
task_key=task["task_key"],
|
||||
agent_id=task["agent_id"],
|
||||
objective=contract.objective,
|
||||
reason=final["reason"],
|
||||
failure_codes=list(check.get("failure_codes") or []),
|
||||
attempts=task["attempt_count"],
|
||||
)
|
||||
task["gap"] = gap
|
||||
task["status"] = TASK_CONTROLLED_GAP
|
||||
self._state.controlled_gaps.append(gap)
|
||||
await self._store.seat_tasks.update_task(task["id"], {
|
||||
"status": TASK_CONTROLLED_GAP,
|
||||
"gap": gap,
|
||||
"review_decision": final,
|
||||
})
|
||||
|
||||
else: # OUTCOME_ASK_USER
|
||||
task["status"] = TASK_EXHAUSTED
|
||||
await self._store.seat_tasks.update_task(task["id"], {
|
||||
"status": TASK_EXHAUSTED,
|
||||
"review_decision": final,
|
||||
})
|
||||
|
||||
return outcome
|
||||
|
||||
# ── backup selection (server-validated) ─────────────────────────────────
|
||||
|
||||
def _backup_agents(self, task: dict[str, Any]) -> list[str]:
|
||||
"""Ordered backup candidates for a task.
|
||||
|
||||
Candidates = server candidate seats minus every agent this task has
|
||||
already used (original + rework owners) minus every agent that is the
|
||||
current owner of ANY other task. A seat owns at most one task at a
|
||||
time — a seat whose own task was accepted must not be double-booked as
|
||||
a backup, because it would then hold two tasks and its deliveries
|
||||
would collide on the (run, agent, turn, stage) idempotency key.
|
||||
"""
|
||||
candidates = list(self._state.candidate_seat_agent_ids or [])
|
||||
used: set[str] = {
|
||||
str(item.get("agent_id") or "")
|
||||
for item in (task.get("history") or [])
|
||||
if str(item.get("agent_id") or "")
|
||||
}
|
||||
used.add(str(task.get("agent_id") or ""))
|
||||
for other in self._tasks:
|
||||
if other.get("task_key") == task.get("task_key"):
|
||||
continue
|
||||
used.add(str(other.get("agent_id") or ""))
|
||||
return [agent_id for agent_id in candidates if agent_id not in used]
|
||||
|
||||
def _pick_backup(self, task: dict[str, Any]) -> str:
|
||||
backups = self._backup_agents(task)
|
||||
if not backups:
|
||||
raise RuntimeError(
|
||||
f"MAF quality loop requested reassignment for task "
|
||||
f"{task.get('task_key')} but no validated backup seat exists"
|
||||
)
|
||||
return backups[0]
|
||||
|
||||
|
||||
class QualityGateExecutor(Executor):
|
||||
"""Stage 4 (R3-Q): phase quality gate between seat tasks and cross-review.
|
||||
|
||||
All required seat tasks must have reached a terminal state (``accepted``
|
||||
or ``controlled_gap``) before the phase may proceed. A required task left
|
||||
non-terminal blocks the phase deterministically (hard run failure — the
|
||||
quality loop guarantees terminal states, so a blocked gate is a safety
|
||||
net, never a normal outcome). Controlled gaps do NOT block the phase:
|
||||
they proceed as ``passed_with_gaps`` and are carried into the final
|
||||
report's risk section.
|
||||
"""
|
||||
|
||||
def __init__(self, adapter: Any, state: RoundtableState) -> None:
|
||||
super().__init__(id="quality_gate")
|
||||
self._adapter = adapter
|
||||
self._state = state
|
||||
|
||||
@handler(input=Any, output=dict)
|
||||
async def handle(self, message: Any, ctx: WorkflowContext) -> None:
|
||||
if self._state.is_completed("quality_gate"):
|
||||
await ctx.send_message({"decision": "passed", "replayed": True})
|
||||
return
|
||||
|
||||
# ask_user path: route to the waiting_input terminal, never to
|
||||
# cross-review / finalizer.
|
||||
if self._state.status == "waiting_input":
|
||||
self._state.mark_completed("quality_gate")
|
||||
await ctx.send_message({"decision": "waiting_input"})
|
||||
return
|
||||
|
||||
statuses = self._state.seat_task_statuses
|
||||
required = {k: v for k, v in statuses.items() if v.get("required")}
|
||||
non_terminal = [
|
||||
k for k, v in required.items() if v.get("status") not in TASK_TERMINAL_STATUSES
|
||||
]
|
||||
# P0-4 safety net: every dispatch assignment must be represented by a
|
||||
# task row. A missing required task blocks the gate exactly like a
|
||||
# non-terminal one — the gate must never pass on a dropped seat.
|
||||
expected_keys: list[str] = []
|
||||
plan = self._state.dispatch_plan or {}
|
||||
for index, assignment in enumerate(plan.get("assignments") or [], start=1):
|
||||
agent_id = str(assignment.get("agent_id") or "").strip()
|
||||
if agent_id:
|
||||
expected_keys.append(f"{index:02d}:{agent_id}")
|
||||
missing = [k for k in expected_keys if k not in statuses]
|
||||
blocked = sorted({*non_terminal, *missing})
|
||||
accepted = sum(1 for v in statuses.values() if v.get("status") == TASK_ACCEPTED)
|
||||
gap_count = sum(1 for v in statuses.values() if v.get("status") == TASK_CONTROLLED_GAP)
|
||||
decision = "blocked" if blocked else ("passed_with_gaps" if gap_count else "passed")
|
||||
|
||||
await self._adapter._emit_critical(self._state.run_id, make_stage_quality_gate(
|
||||
self._state.manager_agent_id,
|
||||
self._state.current_round,
|
||||
decision=decision,
|
||||
total=len(statuses),
|
||||
accepted=accepted,
|
||||
controlled_gap=gap_count,
|
||||
blocked_required=blocked,
|
||||
))
|
||||
|
||||
if blocked:
|
||||
raise RuntimeError(
|
||||
f"MAF quality gate blocked: required seat tasks not terminal or missing: "
|
||||
f"{', '.join(blocked)}"
|
||||
)
|
||||
|
||||
self._state.mark_completed("quality_gate")
|
||||
self._state.stage = "review"
|
||||
await ctx.send_message({"decision": decision, "tasks": dict(statuses)})
|
||||
|
||||
|
||||
# ── helpers ─────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user