sync: 鍚屾灞€鍩熺綉鏈€鏂颁唬鐮?(source master @ b69568e4)

This commit is contained in:
gozeros committed 2026-10-21 14:51:31 +08:00
1 parent 4f7f07504a
commit 8bc8b0e082
1651 files changed
+278599 -11416

No files matched your search

@@ -47,7 +47,18 @@ class ReportCollaborationExecutor:
existing = self._tasks.get(params.run_id)
if existing is not None and not existing.done():
return False
self._tasks[params.run_id] = asyncio.create_task(self.execute(params), name=f"rc-run-{params.run_id}")
task = asyncio.create_task(self.execute(params), name=f"rc-run-{params.run_id}")
def _log_crash(finished: asyncio.Task[str]) -> None:
if not finished.cancelled() and finished.exception() is not None:
logger.error(
"rc executor task crashed for run %s",
params.run_id,
exc_info=finished.exception(),
)
task.add_done_callback(_log_crash)
self._tasks[params.run_id] = task
return True
async def drain(self) -> None:
@@ -10,7 +10,7 @@ from app.report_collaboration.execution.board import ArtifactBoard
from app.report_collaboration.execution.completion import late_output_may_write_completed, require_can_complete_node
from app.report_collaboration.execution.quality_gate import QualityGate, QualityGateContext, requirement_from_session_record
from deerflow.persistence.report_collaboration.base import ReportCollaborationStore
from deerflow.persistence.report_collaboration.codec import iso_now, make_node_run_id
from deerflow.persistence.report_collaboration.codec import _TERMINAL_RUN_STATUSES, iso_now, make_node_run_id
TaskResultKind = Literal["artifact_draft", "clarification_request", "repair_request", "failure", "advice"]
@@ -39,6 +39,8 @@ class TaskAssignment:
agent_profile_snapshot: dict[str, Any]
command_overlay_revision: int
label: str
agent_id: str | None = None
agent_description: str | None = None
angle: str | None = None
command_overlays: list[dict[str, Any]] = field(default_factory=list)
@@ -105,6 +107,42 @@ class TaskLedger:
await self._store.patch_run(run_id, current_phase=phase_id, status="running")
return assignments
async def start_assignment(self, assignment: TaskAssignment) -> bool:
"""Mark one queued assignment running immediately before model execution.
A wave may contain multiple assignments while the current executor runs
them one by one. Keeping this transition separate from queue creation
prevents not-yet-started siblings from being shown as running, while
still making the active member visible for the entire streamed reply.
"""
run = await self._store.get_run(assignment.run_id)
if run is None or run.get("cancel_requested") or run.get("status") in {"cancelled", "completed", "failed"}:
return False
nodes = await self._store.list_node_runs(assignment.run_id)
node = next((item for item in nodes if item["node_run_id"] == assignment.node_run_id), None)
if node is None:
return False
if node.get("status") == "running":
return True
if node.get("status") != "queued":
return False
require_transition("queued", "running")
node["status"] = "running"
node["started_at"] = iso_now()
await self._store.save_node_run(node)
await self._emit(
assignment,
"node.status.changed",
{
"node_id": assignment.node_id,
"node_run_id": assignment.node_run_id,
"status": "running",
"phase_id": assignment.phase_id,
},
)
return True
async def apply_result(self, assignment: TaskAssignment, result: TaskResult) -> ApplyOutcome:
ignored = list(result.suggested_next_node_ids or [])
if result.kind == "advice":
@@ -149,6 +187,15 @@ class TaskLedger:
node["awaiting_input"] = result.clarification or {"reason": "clarification", "question": result.repair_reason or ""}
node["ended_at"] = iso_now()
await self._store.save_node_run(node)
await self._emit(
assignment,
"node.status.changed",
{
"node_id": assignment.node_id,
"node_run_id": assignment.node_run_id,
"status": "awaiting_input",
},
)
await self._store.patch_run(assignment.run_id, status="awaiting_input", session_status="awaiting_input")
return ApplyOutcome(node_status="awaiting_input", run_status="awaiting_input", ignored_coordinator_suggestions=ignored)
@@ -178,6 +225,15 @@ class TaskLedger:
require_transition(node["status"], "validating")
node["status"] = "validating"
await self._store.save_node_run(node)
await self._emit(
assignment,
"node.status.changed",
{
"node_id": assignment.node_id,
"node_run_id": assignment.node_run_id,
"status": "validating",
},
)
if not gate.ok:
if gate.parsed:
rejected = await self._board.submit(
@@ -248,6 +304,15 @@ class TaskLedger:
node["ended_at"] = iso_now()
node["error"] = None
await self._store.save_node_run(node)
await self._emit(
assignment,
"node.status.changed",
{
"node_id": assignment.node_id,
"node_run_id": assignment.node_run_id,
"status": "completed",
},
)
await self._emit(assignment, "artifact.validated", {"artifact_id": artifact["id"], "node_id": assignment.node_id})
if expected == "ReviewDecision" and (gate.parsed or {}).get("verdict") == "revise":
await self._replan_review_targets(assignment, gate.parsed or {})
@@ -314,6 +379,7 @@ class TaskLedger:
command_id: str,
instruction: str,
node_ids: list[str],
cascade_downstream: bool = False,
) -> list[str]:
"""Supersede affected attempts and persist an immutable command overlay.
@@ -321,6 +387,10 @@ class TaskLedger:
``superseded`` and is therefore stored only as superseded evidence.
User-directed rework is not counted against automatic quality-repair
limits; it creates the next monotonic attempt number.
``cascade_downstream=True`` additionally supersedes every transitive
dependent of ``node_ids`` and re-plans it, so the wave loop re-runs
the whole affected sub-DAG in dependency order (设计 §4.2 指定重跑).
"""
plan, run, latest, artifacts = await self._load(run_id)
@@ -407,8 +477,112 @@ class TaskLedger:
active_dedupe_key=str(run["session_id"]),
session_status="running",
)
if cascade_downstream and applied:
applied.extend(
await self._cascade_downstream(
run,
plan,
latest,
artifacts,
roots=[node_id for node_id in node_ids if node_id in applied],
command_id=command_id,
)
)
return applied
async def _cascade_downstream(
self,
run: dict[str, Any],
plan: dict[str, Any],
latest: dict[str, dict[str, Any]],
artifacts: list[dict[str, Any]],
*,
roots: list[str],
command_id: str,
) -> list[str]:
"""Supersede transitive dependents of ``roots`` and re-plan them.
Planned/reopened dependents are left alone — they have produced
nothing yet and pick up the fresh upstream artifacts when queued.
Like the rework target itself, user-directed cascade re-planning is
not budget-limited (monotonic attempt number).
"""
deps = _dependency_map(plan)
specs = _plan_nodes(plan)
dependents: set[str] = set()
changed = True
while changed:
changed = False
for node_id, parents in deps.items():
if node_id in dependents or node_id in roots:
continue
if any(parent in dependents or parent in roots for parent in parents):
dependents.add(node_id)
changed = True
cascaded: list[str] = []
for node_id in sorted(dependents):
node = latest.get(node_id)
spec = specs.get(node_id)
if node is None or spec is None:
continue
status = str(node.get("status") or "")
if status in {"planned", "reopened", "superseded", "cancelled", "cancel_requested", ""}:
continue
require_transition(status, "superseded")
node["status"] = "superseded"
node["ended_at"] = iso_now()
node["awaiting_input"] = None
await self._store.save_node_run(node)
for artifact in artifacts:
if (
artifact.get("node_id") == node_id
and int(artifact.get("attempt") or 0) == int(node.get("attempt") or 0)
and not artifact.get("superseded_by")
):
await self._board.mark(artifact, validation_status="superseded", superseded_by=f"command:{command_id}")
await self._store.append_event(
session_id=str(run["session_id"]),
run_id=str(run["id"]),
event_type="node.status.changed",
data={"node_id": node_id, "node_run_id": node["node_run_id"], "status": "superseded", "reason": "downstream_cascade"},
)
nxt = int(node.get("attempt") or 0) + 1
await self._store.save_node_run(
{
"node_run_id": make_node_run_id(str(run["id"]), node_id, nxt),
"session_id": run["session_id"],
"run_id": run["id"],
"node_id": node_id,
"attempt": nxt,
"status": "planned",
"label": node.get("label") or spec.get("label") or node_id,
"role_display_name": node.get("role_display_name") or spec.get("role_display_name"),
"angle": node.get("angle") or spec.get("angle"),
"contract_json": node.get("contract_json"),
"input_artifact_ids": [],
"command_overlay_revision": int(node.get("command_overlay_revision") or 0),
"agent_snapshot": dict(node.get("agent_snapshot") or {}),
"error": None,
"started_at": None,
"ended_at": None,
}
)
await self._store.append_event(
session_id=str(run["session_id"]),
run_id=str(run["id"]),
event_type="node.attempt.created",
data={
"node_id": node_id,
"node_run_id": make_node_run_id(str(run["id"]), node_id, nxt),
"attempt": nxt,
"reason": "downstream_cascade",
"command_id": command_id,
},
)
cascaded.append(node_id)
return cascaded
@staticmethod
def _add_command_overlay(node: dict[str, Any], *, command_id: str, instruction: str) -> None:
snapshot = dict(node.get("agent_snapshot") or {})
@@ -595,6 +769,8 @@ class TaskLedger:
agent_profile_snapshot=snapshot,
command_overlay_revision=int(current.get("command_overlay_revision") or 0),
label=str(spec.get("label") or node_id),
agent_id=str(spec.get("agent_id") or "") or None,
agent_description=str(spec.get("agent_description") or "") or None,
angle=spec.get("angle"),
command_overlays=[dict(item) for item in snapshot.get("command_overlays") or [] if isinstance(item, dict)],
)
@@ -622,8 +798,18 @@ class TaskLedger:
return False
return True
async def refresh_run_terminal(self, run_id: str) -> str:
"""Public terminal-state evaluation; safe to call repeatedly."""
return await self._refresh_run_terminal(run_id)
async def _refresh_run_terminal(self, run_id: str) -> str:
_plan, run, latest, _artifacts = await self._load(run_id)
plan, run, latest, _artifacts = await self._load(run_id)
run_status = str(run.get("status") or "running")
if run_status in _TERMINAL_RUN_STATUSES:
# Idempotency guard: never re-finalize (the finalizer would run
# twice when the executor's wave loop re-checks after completion).
return run_status
statuses = [item["status"] for item in latest.values()]
if statuses and all(status == "completed" for status in statuses):
from app.report_collaboration.reporting.finalizer import ReportFinalizationError, ReportFinalizer
@@ -636,11 +822,32 @@ class TaskLedger:
await self._store.patch_run(run_id, status="completed", active_dedupe_key=None, session_status="completed")
return "completed"
if any(status in _BLOCKING or status == "queued" for status in statuses):
return str(run.get("status") or "running")
return run_status
if any(status == "failed" for status in statuses):
await self._store.patch_run(run_id, status="failed", active_dedupe_key=None, session_status="failed")
await self._fail_run(run_id, run, reason="node_failed")
return "failed"
return str(run.get("status") or "running")
# Zombie guard: an ``interrupted`` node whose attempt budget is
# exhausted can never run again — the ledger only queues
# planned/reopened attempts and recovery only opens a new attempt
# while attempt < max_attempts. Without this, the run would stay
# "running" forever while the dispatcher keeps re-claiming it.
specs = _plan_nodes(plan)
for node_id, item in latest.items():
if str(item.get("status")) != "interrupted":
continue
if int(item.get("attempt") or 1) >= _node_max_attempts(specs.get(node_id), item):
await self._fail_run(run_id, run, reason="node_attempts_exhausted", node_id=node_id)
return "failed"
return run_status
async def _fail_run(self, run_id: str, run: dict[str, Any], *, reason: str, node_id: str | None = None) -> None:
await self._store.patch_run(run_id, status="failed", active_dedupe_key=None, session_status="failed")
await self._store.append_event(
session_id=str(run.get("session_id") or ""),
run_id=run_id,
event_type="run.status.changed",
data={"status": "failed", "reason": reason, **({"node_id": node_id} if node_id else {})},
)
async def _load(self, run_id: str) -> tuple[dict[str, Any], dict[str, Any], dict[str, dict[str, Any]], list[dict[str, Any]]]:
run = await self._store.get_run(run_id)
@@ -666,6 +873,20 @@ def _plan_nodes(plan: dict[str, Any]) -> dict[str, dict[str, Any]]:
return {str(item["id"]): item for item in (plan.get("nodes") or []) if item.get("id")}
def _node_max_attempts(spec: dict[str, Any] | None, node_record: dict[str, Any]) -> int:
"""Same precedence as ``_queue_node``: node contract, then plan spec, then 3."""
contract = node_record.get("contract_json") or {}
if isinstance(contract, str):
try:
import json as _json
contract = _json.loads(contract)
except Exception:
contract = {}
return int(contract.get("max_attempts") or (spec or {}).get("max_attempts") or 3)
def _dependency_map(plan: dict[str, Any]) -> dict[str, list[str]]:
deps: dict[str, list[str]] = {node_id: list(spec.get("depends_on") or []) for node_id, spec in _plan_nodes(plan).items()}
for edge in plan.get("edges") or []:
@@ -2,6 +2,8 @@
from __future__ import annotations
import asyncio
import logging
from collections.abc import Callable
from typing import Any, Protocol
from uuid import uuid4
@@ -13,6 +15,8 @@ from app.report_collaboration.execution.live_hub import CoalescingEventSink
from deerflow.persistence.report_collaboration.base import ReportCollaborationStore
from deerflow.persistence.report_collaboration.codec import iso_now
logger = logging.getLogger(__name__)
class MemberKernel(Protocol):
async def run(self, assignment: TaskAssignment, *, context: dict[str, Any]) -> TaskResult:
@@ -86,7 +90,21 @@ class ControlledMemberRunner:
ctx["permission"] = snapshot
ctx["agent_run_id"] = agent_run_id
ctx["command_overlays"] = list(assignment.command_overlays)
result = await self._kernel.run(assignment, context=ctx)
try:
result = await self._kernel.run(assignment, context=ctx)
except asyncio.CancelledError:
raise
except Exception as exc: # noqa: BLE001 - member failures are workflow results
logger.exception(
"report collaboration member failed (run=%s node=%s attempt=%s)",
assignment.run_id,
assignment.node_id,
assignment.attempt,
)
result = TaskResult(
kind="failure",
error=f"MEMBER_EXECUTION_ERROR:{type(exc).__name__}:{exc}",
)
await self._mirror_reply(assignment, agent_run_id, result)
if result.kind == "clarification_request":
agent_status = "waiting_input"
@@ -195,9 +213,14 @@ class WaveExecutor:
await self._command_processor.process_safe_point(run_id)
wave = await self._ledger.ready_assignments(run_id)
if not wave:
run = await self._store.get_run(run_id)
return str((run or {}).get("status") or "queued")
# Nothing runnable. Evaluate terminal state: all-completed
# finalizes, exhausted-interrupted nodes fail the run (zombie
# guard). Idempotent when the run already reached a terminal
# status via apply_result.
return await self._ledger.refresh_run_terminal(run_id)
for assignment in wave:
if not await self._ledger.start_assignment(assignment):
continue
self.dispatched.append(assignment.node_id)
result = await self._runner.run(assignment)
if self._command_processor is not None: