sync: 鍚屾灞€鍩熺綉鏈€鏂颁唬鐮?(source master @ b69568e4)
This commit is contained in:
1 parent
4f7f07504a
commit
8bc8b0e082
1651 files changed
+278599
-11416
No files matched your search
@@ -47,7 +47,18 @@ class ReportCollaborationExecutor:
|
||||
existing = self._tasks.get(params.run_id)
|
||||
if existing is not None and not existing.done():
|
||||
return False
|
||||
self._tasks[params.run_id] = asyncio.create_task(self.execute(params), name=f"rc-run-{params.run_id}")
|
||||
task = asyncio.create_task(self.execute(params), name=f"rc-run-{params.run_id}")
|
||||
|
||||
def _log_crash(finished: asyncio.Task[str]) -> None:
|
||||
if not finished.cancelled() and finished.exception() is not None:
|
||||
logger.error(
|
||||
"rc executor task crashed for run %s",
|
||||
params.run_id,
|
||||
exc_info=finished.exception(),
|
||||
)
|
||||
|
||||
task.add_done_callback(_log_crash)
|
||||
self._tasks[params.run_id] = task
|
||||
return True
|
||||
|
||||
async def drain(self) -> None:
|
||||
|
||||
@@ -10,7 +10,7 @@ from app.report_collaboration.execution.board import ArtifactBoard
|
||||
from app.report_collaboration.execution.completion import late_output_may_write_completed, require_can_complete_node
|
||||
from app.report_collaboration.execution.quality_gate import QualityGate, QualityGateContext, requirement_from_session_record
|
||||
from deerflow.persistence.report_collaboration.base import ReportCollaborationStore
|
||||
from deerflow.persistence.report_collaboration.codec import iso_now, make_node_run_id
|
||||
from deerflow.persistence.report_collaboration.codec import _TERMINAL_RUN_STATUSES, iso_now, make_node_run_id
|
||||
|
||||
TaskResultKind = Literal["artifact_draft", "clarification_request", "repair_request", "failure", "advice"]
|
||||
|
||||
@@ -39,6 +39,8 @@ class TaskAssignment:
|
||||
agent_profile_snapshot: dict[str, Any]
|
||||
command_overlay_revision: int
|
||||
label: str
|
||||
agent_id: str | None = None
|
||||
agent_description: str | None = None
|
||||
angle: str | None = None
|
||||
command_overlays: list[dict[str, Any]] = field(default_factory=list)
|
||||
|
||||
@@ -105,6 +107,42 @@ class TaskLedger:
|
||||
await self._store.patch_run(run_id, current_phase=phase_id, status="running")
|
||||
return assignments
|
||||
|
||||
async def start_assignment(self, assignment: TaskAssignment) -> bool:
|
||||
"""Mark one queued assignment running immediately before model execution.
|
||||
|
||||
A wave may contain multiple assignments while the current executor runs
|
||||
them one by one. Keeping this transition separate from queue creation
|
||||
prevents not-yet-started siblings from being shown as running, while
|
||||
still making the active member visible for the entire streamed reply.
|
||||
"""
|
||||
|
||||
run = await self._store.get_run(assignment.run_id)
|
||||
if run is None or run.get("cancel_requested") or run.get("status") in {"cancelled", "completed", "failed"}:
|
||||
return False
|
||||
nodes = await self._store.list_node_runs(assignment.run_id)
|
||||
node = next((item for item in nodes if item["node_run_id"] == assignment.node_run_id), None)
|
||||
if node is None:
|
||||
return False
|
||||
if node.get("status") == "running":
|
||||
return True
|
||||
if node.get("status") != "queued":
|
||||
return False
|
||||
require_transition("queued", "running")
|
||||
node["status"] = "running"
|
||||
node["started_at"] = iso_now()
|
||||
await self._store.save_node_run(node)
|
||||
await self._emit(
|
||||
assignment,
|
||||
"node.status.changed",
|
||||
{
|
||||
"node_id": assignment.node_id,
|
||||
"node_run_id": assignment.node_run_id,
|
||||
"status": "running",
|
||||
"phase_id": assignment.phase_id,
|
||||
},
|
||||
)
|
||||
return True
|
||||
|
||||
async def apply_result(self, assignment: TaskAssignment, result: TaskResult) -> ApplyOutcome:
|
||||
ignored = list(result.suggested_next_node_ids or [])
|
||||
if result.kind == "advice":
|
||||
@@ -149,6 +187,15 @@ class TaskLedger:
|
||||
node["awaiting_input"] = result.clarification or {"reason": "clarification", "question": result.repair_reason or ""}
|
||||
node["ended_at"] = iso_now()
|
||||
await self._store.save_node_run(node)
|
||||
await self._emit(
|
||||
assignment,
|
||||
"node.status.changed",
|
||||
{
|
||||
"node_id": assignment.node_id,
|
||||
"node_run_id": assignment.node_run_id,
|
||||
"status": "awaiting_input",
|
||||
},
|
||||
)
|
||||
await self._store.patch_run(assignment.run_id, status="awaiting_input", session_status="awaiting_input")
|
||||
return ApplyOutcome(node_status="awaiting_input", run_status="awaiting_input", ignored_coordinator_suggestions=ignored)
|
||||
|
||||
@@ -178,6 +225,15 @@ class TaskLedger:
|
||||
require_transition(node["status"], "validating")
|
||||
node["status"] = "validating"
|
||||
await self._store.save_node_run(node)
|
||||
await self._emit(
|
||||
assignment,
|
||||
"node.status.changed",
|
||||
{
|
||||
"node_id": assignment.node_id,
|
||||
"node_run_id": assignment.node_run_id,
|
||||
"status": "validating",
|
||||
},
|
||||
)
|
||||
if not gate.ok:
|
||||
if gate.parsed:
|
||||
rejected = await self._board.submit(
|
||||
@@ -248,6 +304,15 @@ class TaskLedger:
|
||||
node["ended_at"] = iso_now()
|
||||
node["error"] = None
|
||||
await self._store.save_node_run(node)
|
||||
await self._emit(
|
||||
assignment,
|
||||
"node.status.changed",
|
||||
{
|
||||
"node_id": assignment.node_id,
|
||||
"node_run_id": assignment.node_run_id,
|
||||
"status": "completed",
|
||||
},
|
||||
)
|
||||
await self._emit(assignment, "artifact.validated", {"artifact_id": artifact["id"], "node_id": assignment.node_id})
|
||||
if expected == "ReviewDecision" and (gate.parsed or {}).get("verdict") == "revise":
|
||||
await self._replan_review_targets(assignment, gate.parsed or {})
|
||||
@@ -314,6 +379,7 @@ class TaskLedger:
|
||||
command_id: str,
|
||||
instruction: str,
|
||||
node_ids: list[str],
|
||||
cascade_downstream: bool = False,
|
||||
) -> list[str]:
|
||||
"""Supersede affected attempts and persist an immutable command overlay.
|
||||
|
||||
@@ -321,6 +387,10 @@ class TaskLedger:
|
||||
``superseded`` and is therefore stored only as superseded evidence.
|
||||
User-directed rework is not counted against automatic quality-repair
|
||||
limits; it creates the next monotonic attempt number.
|
||||
|
||||
``cascade_downstream=True`` additionally supersedes every transitive
|
||||
dependent of ``node_ids`` and re-plans it, so the wave loop re-runs
|
||||
the whole affected sub-DAG in dependency order (设计 §4.2 指定重跑).
|
||||
"""
|
||||
|
||||
plan, run, latest, artifacts = await self._load(run_id)
|
||||
@@ -407,8 +477,112 @@ class TaskLedger:
|
||||
active_dedupe_key=str(run["session_id"]),
|
||||
session_status="running",
|
||||
)
|
||||
if cascade_downstream and applied:
|
||||
applied.extend(
|
||||
await self._cascade_downstream(
|
||||
run,
|
||||
plan,
|
||||
latest,
|
||||
artifacts,
|
||||
roots=[node_id for node_id in node_ids if node_id in applied],
|
||||
command_id=command_id,
|
||||
)
|
||||
)
|
||||
return applied
|
||||
|
||||
async def _cascade_downstream(
|
||||
self,
|
||||
run: dict[str, Any],
|
||||
plan: dict[str, Any],
|
||||
latest: dict[str, dict[str, Any]],
|
||||
artifacts: list[dict[str, Any]],
|
||||
*,
|
||||
roots: list[str],
|
||||
command_id: str,
|
||||
) -> list[str]:
|
||||
"""Supersede transitive dependents of ``roots`` and re-plan them.
|
||||
|
||||
Planned/reopened dependents are left alone — they have produced
|
||||
nothing yet and pick up the fresh upstream artifacts when queued.
|
||||
Like the rework target itself, user-directed cascade re-planning is
|
||||
not budget-limited (monotonic attempt number).
|
||||
"""
|
||||
|
||||
deps = _dependency_map(plan)
|
||||
specs = _plan_nodes(plan)
|
||||
dependents: set[str] = set()
|
||||
changed = True
|
||||
while changed:
|
||||
changed = False
|
||||
for node_id, parents in deps.items():
|
||||
if node_id in dependents or node_id in roots:
|
||||
continue
|
||||
if any(parent in dependents or parent in roots for parent in parents):
|
||||
dependents.add(node_id)
|
||||
changed = True
|
||||
cascaded: list[str] = []
|
||||
for node_id in sorted(dependents):
|
||||
node = latest.get(node_id)
|
||||
spec = specs.get(node_id)
|
||||
if node is None or spec is None:
|
||||
continue
|
||||
status = str(node.get("status") or "")
|
||||
if status in {"planned", "reopened", "superseded", "cancelled", "cancel_requested", ""}:
|
||||
continue
|
||||
require_transition(status, "superseded")
|
||||
node["status"] = "superseded"
|
||||
node["ended_at"] = iso_now()
|
||||
node["awaiting_input"] = None
|
||||
await self._store.save_node_run(node)
|
||||
for artifact in artifacts:
|
||||
if (
|
||||
artifact.get("node_id") == node_id
|
||||
and int(artifact.get("attempt") or 0) == int(node.get("attempt") or 0)
|
||||
and not artifact.get("superseded_by")
|
||||
):
|
||||
await self._board.mark(artifact, validation_status="superseded", superseded_by=f"command:{command_id}")
|
||||
await self._store.append_event(
|
||||
session_id=str(run["session_id"]),
|
||||
run_id=str(run["id"]),
|
||||
event_type="node.status.changed",
|
||||
data={"node_id": node_id, "node_run_id": node["node_run_id"], "status": "superseded", "reason": "downstream_cascade"},
|
||||
)
|
||||
nxt = int(node.get("attempt") or 0) + 1
|
||||
await self._store.save_node_run(
|
||||
{
|
||||
"node_run_id": make_node_run_id(str(run["id"]), node_id, nxt),
|
||||
"session_id": run["session_id"],
|
||||
"run_id": run["id"],
|
||||
"node_id": node_id,
|
||||
"attempt": nxt,
|
||||
"status": "planned",
|
||||
"label": node.get("label") or spec.get("label") or node_id,
|
||||
"role_display_name": node.get("role_display_name") or spec.get("role_display_name"),
|
||||
"angle": node.get("angle") or spec.get("angle"),
|
||||
"contract_json": node.get("contract_json"),
|
||||
"input_artifact_ids": [],
|
||||
"command_overlay_revision": int(node.get("command_overlay_revision") or 0),
|
||||
"agent_snapshot": dict(node.get("agent_snapshot") or {}),
|
||||
"error": None,
|
||||
"started_at": None,
|
||||
"ended_at": None,
|
||||
}
|
||||
)
|
||||
await self._store.append_event(
|
||||
session_id=str(run["session_id"]),
|
||||
run_id=str(run["id"]),
|
||||
event_type="node.attempt.created",
|
||||
data={
|
||||
"node_id": node_id,
|
||||
"node_run_id": make_node_run_id(str(run["id"]), node_id, nxt),
|
||||
"attempt": nxt,
|
||||
"reason": "downstream_cascade",
|
||||
"command_id": command_id,
|
||||
},
|
||||
)
|
||||
cascaded.append(node_id)
|
||||
return cascaded
|
||||
|
||||
@staticmethod
|
||||
def _add_command_overlay(node: dict[str, Any], *, command_id: str, instruction: str) -> None:
|
||||
snapshot = dict(node.get("agent_snapshot") or {})
|
||||
@@ -595,6 +769,8 @@ class TaskLedger:
|
||||
agent_profile_snapshot=snapshot,
|
||||
command_overlay_revision=int(current.get("command_overlay_revision") or 0),
|
||||
label=str(spec.get("label") or node_id),
|
||||
agent_id=str(spec.get("agent_id") or "") or None,
|
||||
agent_description=str(spec.get("agent_description") or "") or None,
|
||||
angle=spec.get("angle"),
|
||||
command_overlays=[dict(item) for item in snapshot.get("command_overlays") or [] if isinstance(item, dict)],
|
||||
)
|
||||
@@ -622,8 +798,18 @@ class TaskLedger:
|
||||
return False
|
||||
return True
|
||||
|
||||
async def refresh_run_terminal(self, run_id: str) -> str:
|
||||
"""Public terminal-state evaluation; safe to call repeatedly."""
|
||||
|
||||
return await self._refresh_run_terminal(run_id)
|
||||
|
||||
async def _refresh_run_terminal(self, run_id: str) -> str:
|
||||
_plan, run, latest, _artifacts = await self._load(run_id)
|
||||
plan, run, latest, _artifacts = await self._load(run_id)
|
||||
run_status = str(run.get("status") or "running")
|
||||
if run_status in _TERMINAL_RUN_STATUSES:
|
||||
# Idempotency guard: never re-finalize (the finalizer would run
|
||||
# twice when the executor's wave loop re-checks after completion).
|
||||
return run_status
|
||||
statuses = [item["status"] for item in latest.values()]
|
||||
if statuses and all(status == "completed" for status in statuses):
|
||||
from app.report_collaboration.reporting.finalizer import ReportFinalizationError, ReportFinalizer
|
||||
@@ -636,11 +822,32 @@ class TaskLedger:
|
||||
await self._store.patch_run(run_id, status="completed", active_dedupe_key=None, session_status="completed")
|
||||
return "completed"
|
||||
if any(status in _BLOCKING or status == "queued" for status in statuses):
|
||||
return str(run.get("status") or "running")
|
||||
return run_status
|
||||
if any(status == "failed" for status in statuses):
|
||||
await self._store.patch_run(run_id, status="failed", active_dedupe_key=None, session_status="failed")
|
||||
await self._fail_run(run_id, run, reason="node_failed")
|
||||
return "failed"
|
||||
return str(run.get("status") or "running")
|
||||
# Zombie guard: an ``interrupted`` node whose attempt budget is
|
||||
# exhausted can never run again — the ledger only queues
|
||||
# planned/reopened attempts and recovery only opens a new attempt
|
||||
# while attempt < max_attempts. Without this, the run would stay
|
||||
# "running" forever while the dispatcher keeps re-claiming it.
|
||||
specs = _plan_nodes(plan)
|
||||
for node_id, item in latest.items():
|
||||
if str(item.get("status")) != "interrupted":
|
||||
continue
|
||||
if int(item.get("attempt") or 1) >= _node_max_attempts(specs.get(node_id), item):
|
||||
await self._fail_run(run_id, run, reason="node_attempts_exhausted", node_id=node_id)
|
||||
return "failed"
|
||||
return run_status
|
||||
|
||||
async def _fail_run(self, run_id: str, run: dict[str, Any], *, reason: str, node_id: str | None = None) -> None:
|
||||
await self._store.patch_run(run_id, status="failed", active_dedupe_key=None, session_status="failed")
|
||||
await self._store.append_event(
|
||||
session_id=str(run.get("session_id") or ""),
|
||||
run_id=run_id,
|
||||
event_type="run.status.changed",
|
||||
data={"status": "failed", "reason": reason, **({"node_id": node_id} if node_id else {})},
|
||||
)
|
||||
|
||||
async def _load(self, run_id: str) -> tuple[dict[str, Any], dict[str, Any], dict[str, dict[str, Any]], list[dict[str, Any]]]:
|
||||
run = await self._store.get_run(run_id)
|
||||
@@ -666,6 +873,20 @@ def _plan_nodes(plan: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
||||
return {str(item["id"]): item for item in (plan.get("nodes") or []) if item.get("id")}
|
||||
|
||||
|
||||
def _node_max_attempts(spec: dict[str, Any] | None, node_record: dict[str, Any]) -> int:
|
||||
"""Same precedence as ``_queue_node``: node contract, then plan spec, then 3."""
|
||||
|
||||
contract = node_record.get("contract_json") or {}
|
||||
if isinstance(contract, str):
|
||||
try:
|
||||
import json as _json
|
||||
|
||||
contract = _json.loads(contract)
|
||||
except Exception:
|
||||
contract = {}
|
||||
return int(contract.get("max_attempts") or (spec or {}).get("max_attempts") or 3)
|
||||
|
||||
|
||||
def _dependency_map(plan: dict[str, Any]) -> dict[str, list[str]]:
|
||||
deps: dict[str, list[str]] = {node_id: list(spec.get("depends_on") or []) for node_id, spec in _plan_nodes(plan).items()}
|
||||
for edge in plan.get("edges") or []:
|
||||
|
||||
+26
-3
@@ -2,6 +2,8 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections.abc import Callable
|
||||
from typing import Any, Protocol
|
||||
from uuid import uuid4
|
||||
@@ -13,6 +15,8 @@ from app.report_collaboration.execution.live_hub import CoalescingEventSink
|
||||
from deerflow.persistence.report_collaboration.base import ReportCollaborationStore
|
||||
from deerflow.persistence.report_collaboration.codec import iso_now
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MemberKernel(Protocol):
|
||||
async def run(self, assignment: TaskAssignment, *, context: dict[str, Any]) -> TaskResult:
|
||||
@@ -86,7 +90,21 @@ class ControlledMemberRunner:
|
||||
ctx["permission"] = snapshot
|
||||
ctx["agent_run_id"] = agent_run_id
|
||||
ctx["command_overlays"] = list(assignment.command_overlays)
|
||||
result = await self._kernel.run(assignment, context=ctx)
|
||||
try:
|
||||
result = await self._kernel.run(assignment, context=ctx)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001 - member failures are workflow results
|
||||
logger.exception(
|
||||
"report collaboration member failed (run=%s node=%s attempt=%s)",
|
||||
assignment.run_id,
|
||||
assignment.node_id,
|
||||
assignment.attempt,
|
||||
)
|
||||
result = TaskResult(
|
||||
kind="failure",
|
||||
error=f"MEMBER_EXECUTION_ERROR:{type(exc).__name__}:{exc}",
|
||||
)
|
||||
await self._mirror_reply(assignment, agent_run_id, result)
|
||||
if result.kind == "clarification_request":
|
||||
agent_status = "waiting_input"
|
||||
@@ -195,9 +213,14 @@ class WaveExecutor:
|
||||
await self._command_processor.process_safe_point(run_id)
|
||||
wave = await self._ledger.ready_assignments(run_id)
|
||||
if not wave:
|
||||
run = await self._store.get_run(run_id)
|
||||
return str((run or {}).get("status") or "queued")
|
||||
# Nothing runnable. Evaluate terminal state: all-completed
|
||||
# finalizes, exhausted-interrupted nodes fail the run (zombie
|
||||
# guard). Idempotent when the run already reached a terminal
|
||||
# status via apply_result.
|
||||
return await self._ledger.refresh_run_terminal(run_id)
|
||||
for assignment in wave:
|
||||
if not await self._ledger.start_assignment(assignment):
|
||||
continue
|
||||
self.dispatched.append(assignment.node_id)
|
||||
result = await self._runner.run(assignment)
|
||||
if self._command_processor is not None:
|
||||
|
||||
Reference in new issue
Block a user