sync update from LAN repo (source dhk-1 @ 3b1ec5b4)
This commit is contained in:
1 parent
da6521b6cb
commit
264e2200d6
995 files changed
+148105
-13399
No files matched your search
@@ -6,7 +6,7 @@ import logging
|
||||
import threading
|
||||
import uuid
|
||||
from collections.abc import Callable, Coroutine
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from concurrent.futures import Future, InvalidStateError
|
||||
from concurrent.futures import TimeoutError as FuturesTimeoutError
|
||||
from contextvars import Context, copy_context
|
||||
from dataclasses import dataclass, field
|
||||
@@ -22,7 +22,11 @@ from langchain_core.runnables import RunnableConfig
|
||||
from deerflow.agents.thread_state import SandboxState, ThreadDataState, ThreadState
|
||||
from deerflow.config import get_app_config
|
||||
from deerflow.config.app_config import AppConfig
|
||||
from deerflow.config.runtime_contention_config import get_runtime_contention_config
|
||||
from deerflow.models import create_chat_model
|
||||
from deerflow.runtime.fair_queue import FairBackgroundScheduler, QueueFullError
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
from deerflow.sandbox.prompt_policy import get_agent_runtime_policy
|
||||
from deerflow.subagents.config import SubagentConfig, resolve_subagent_model_name
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -69,6 +73,8 @@ class SubagentResult:
|
||||
completed_at: datetime | None = None
|
||||
ai_messages: list[dict[str, Any]] | None = None
|
||||
cancel_event: threading.Event = field(default_factory=threading.Event, repr=False)
|
||||
execution_finished: threading.Event = field(default_factory=threading.Event, repr=False)
|
||||
cleanup_requested: bool = False
|
||||
|
||||
def __post_init__(self):
|
||||
"""Initialize mutable defaults."""
|
||||
@@ -81,7 +87,18 @@ _background_tasks: dict[str, SubagentResult] = {}
|
||||
_background_tasks_lock = threading.Lock()
|
||||
|
||||
# Thread pool for background task scheduling and orchestration
|
||||
_scheduler_pool = ThreadPoolExecutor(max_workers=3, thread_name_prefix="subagent-scheduler-")
|
||||
_scheduler_pool: FairBackgroundScheduler | None = None
|
||||
_scheduler_init_lock = threading.Lock()
|
||||
|
||||
|
||||
def _get_scheduler() -> FairBackgroundScheduler:
|
||||
global _scheduler_pool
|
||||
with _scheduler_init_lock:
|
||||
if _scheduler_pool is None:
|
||||
limits = get_runtime_contention_config()
|
||||
_scheduler_pool = FairBackgroundScheduler(limits.subagent_concurrency, limits.subagent_per_user, limits.queue_limit)
|
||||
atexit.register(_scheduler_pool.shutdown)
|
||||
return _scheduler_pool
|
||||
|
||||
# Persistent event loop for isolated subagent executions triggered from an
|
||||
# already-running parent loop. Reusing one long-lived loop avoids creating a
|
||||
@@ -176,14 +193,51 @@ def _get_isolated_subagent_loop() -> asyncio.AbstractEventLoop:
|
||||
def _submit_to_isolated_loop_in_context(
|
||||
context: Context,
|
||||
coro_factory: Callable[[], Coroutine[Any, Any, SubagentResult]],
|
||||
*,
|
||||
on_finished: Callable[[], None] | None = None,
|
||||
) -> Future[SubagentResult]:
|
||||
"""Submit a coroutine to the isolated loop while preserving ContextVar state."""
|
||||
return context.run(
|
||||
lambda: asyncio.run_coroutine_threadsafe(
|
||||
coro_factory(),
|
||||
_get_isolated_subagent_loop(),
|
||||
)
|
||||
)
|
||||
"""Bridge completion and real task cancellation, including before first step."""
|
||||
future: Future[SubagentResult] = Future()
|
||||
loop = _get_isolated_subagent_loop()
|
||||
|
||||
def schedule():
|
||||
if future.cancelled():
|
||||
if on_finished:
|
||||
on_finished()
|
||||
return
|
||||
try:
|
||||
task = loop.create_task(coro_factory(), context=context.copy())
|
||||
except BaseException as exc:
|
||||
if on_finished:
|
||||
on_finished()
|
||||
if not future.done():
|
||||
try:
|
||||
future.set_exception(exc)
|
||||
except InvalidStateError:
|
||||
pass # Cancelled concurrently by the caller.
|
||||
return
|
||||
|
||||
def completed(done):
|
||||
if on_finished:
|
||||
on_finished()
|
||||
if done.cancelled():
|
||||
future.cancel()
|
||||
else:
|
||||
error = done.exception()
|
||||
if not future.done():
|
||||
try:
|
||||
if error is not None:
|
||||
future.set_exception(error)
|
||||
else:
|
||||
future.set_result(done.result())
|
||||
except InvalidStateError:
|
||||
pass # Caller cancellation raced with loop completion.
|
||||
|
||||
task.add_done_callback(completed)
|
||||
future.add_done_callback(lambda result: loop.call_soon_threadsafe(task.cancel) if result.cancelled() and not task.done() else None)
|
||||
|
||||
loop.call_soon_threadsafe(schedule, context=context.copy())
|
||||
return future
|
||||
|
||||
|
||||
def _filter_tools(
|
||||
@@ -236,6 +290,8 @@ class SubagentExecutor:
|
||||
thread_data: ThreadDataState | None = None,
|
||||
thread_id: str | None = None,
|
||||
trace_id: str | None = None,
|
||||
*,
|
||||
write_file_compatibility: bool | None = None,
|
||||
):
|
||||
"""Initialize the executor.
|
||||
|
||||
@@ -264,6 +320,7 @@ class SubagentExecutor:
|
||||
self.sandbox_state = sandbox_state
|
||||
self.thread_data = thread_data
|
||||
self.thread_id = thread_id
|
||||
self.write_file_compatibility = True if write_file_compatibility else None
|
||||
# Generate trace_id if not provided (for top-level calls)
|
||||
self.trace_id = trace_id or str(uuid.uuid4())[:8]
|
||||
|
||||
@@ -281,7 +338,12 @@ class SubagentExecutor:
|
||||
app_config = self.app_config or get_app_config()
|
||||
if self.model_name is None:
|
||||
self.model_name = resolve_subagent_model_name(self.config, self.parent_model, app_config=app_config)
|
||||
model = create_chat_model(name=self.model_name, thinking_enabled=False, app_config=app_config)
|
||||
model = create_chat_model(
|
||||
name=self.model_name,
|
||||
thinking_enabled=False,
|
||||
app_config=app_config,
|
||||
write_file_compatibility=self.write_file_compatibility,
|
||||
)
|
||||
|
||||
from deerflow.agents.middlewares.tool_error_handling_middleware import build_subagent_runtime_middlewares
|
||||
|
||||
@@ -292,7 +354,7 @@ class SubagentExecutor:
|
||||
model=model,
|
||||
tools=self.tools,
|
||||
middleware=middlewares,
|
||||
system_prompt=self.config.system_prompt,
|
||||
system_prompt=self.config.system_prompt + get_agent_runtime_policy(),
|
||||
state_schema=ThreadState,
|
||||
)
|
||||
|
||||
@@ -673,46 +735,82 @@ class SubagentExecutor:
|
||||
parent_context = copy_context()
|
||||
|
||||
# Submit to scheduler pool
|
||||
def run_task():
|
||||
def run_task(remaining_seconds):
|
||||
with _background_tasks_lock:
|
||||
_background_tasks[task_id].status = SubagentStatus.RUNNING
|
||||
_background_tasks[task_id].started_at = datetime.now()
|
||||
result_holder = _background_tasks[task_id]
|
||||
result.status = SubagentStatus.RUNNING
|
||||
result.started_at = datetime.now()
|
||||
result_holder = result
|
||||
|
||||
execution_future = None
|
||||
|
||||
async def execute_until_stopped():
|
||||
try:
|
||||
return await self._aexecute(task, result_holder)
|
||||
finally:
|
||||
result_holder.execution_finished.set()
|
||||
|
||||
try:
|
||||
# Submit execution directly to the persistent isolated loop so the
|
||||
# background path does not create a temporary loop via execute().
|
||||
execution_future = _submit_to_isolated_loop_in_context(
|
||||
parent_context,
|
||||
lambda: self._aexecute(task, result_holder),
|
||||
execute_until_stopped,
|
||||
on_finished=result_holder.execution_finished.set,
|
||||
)
|
||||
try:
|
||||
# Wait for execution with timeout
|
||||
exec_result = execution_future.result(timeout=self.config.timeout_seconds)
|
||||
exec_result = execution_future.result(timeout=remaining_seconds)
|
||||
with _background_tasks_lock:
|
||||
_background_tasks[task_id].status = exec_result.status
|
||||
_background_tasks[task_id].result = exec_result.result
|
||||
_background_tasks[task_id].error = exec_result.error
|
||||
_background_tasks[task_id].completed_at = datetime.now()
|
||||
_background_tasks[task_id].ai_messages = exec_result.ai_messages
|
||||
result.status = exec_result.status
|
||||
result.result = exec_result.result
|
||||
result.error = exec_result.error
|
||||
result.completed_at = datetime.now()
|
||||
result.ai_messages = exec_result.ai_messages
|
||||
except FuturesTimeoutError:
|
||||
logger.error(f"[trace={self.trace_id}] Subagent {self.config.name} execution timed out after {self.config.timeout_seconds}s")
|
||||
with _background_tasks_lock:
|
||||
if _background_tasks[task_id].status == SubagentStatus.RUNNING:
|
||||
_background_tasks[task_id].status = SubagentStatus.TIMED_OUT
|
||||
_background_tasks[task_id].error = f"Execution timed out after {self.config.timeout_seconds} seconds"
|
||||
_background_tasks[task_id].completed_at = datetime.now()
|
||||
if result.status == SubagentStatus.RUNNING:
|
||||
result.status = SubagentStatus.TIMED_OUT
|
||||
result.error = f"Execution timed out after {self.config.timeout_seconds} seconds"
|
||||
result.completed_at = datetime.now()
|
||||
# Signal cooperative cancellation and cancel the future
|
||||
result_holder.cancel_event.set()
|
||||
execution_future.cancel()
|
||||
except Exception as e:
|
||||
logger.exception(f"[trace={self.trace_id}] Subagent {self.config.name} async execution failed")
|
||||
with _background_tasks_lock:
|
||||
_background_tasks[task_id].status = SubagentStatus.FAILED
|
||||
_background_tasks[task_id].error = str(e)
|
||||
_background_tasks[task_id].completed_at = datetime.now()
|
||||
|
||||
_scheduler_pool.submit(run_task)
|
||||
result.status = SubagentStatus.FAILED
|
||||
result.error = str(e)
|
||||
result.completed_at = datetime.now()
|
||||
|
||||
finally:
|
||||
if execution_future is None:
|
||||
result_holder.execution_finished.set()
|
||||
else:
|
||||
# A concurrent Future's cancellation is not proof that its
|
||||
# coroutine has stopped. Keep the admission slot until ack.
|
||||
result_holder.execution_finished.wait()
|
||||
if result_holder.cleanup_requested:
|
||||
cleanup_background_task(task_id)
|
||||
|
||||
def expired(cancelled):
|
||||
with _background_tasks_lock:
|
||||
result.status = SubagentStatus.CANCELLED if cancelled else SubagentStatus.TIMED_OUT
|
||||
result.error = "Cancelled while queued" if cancelled else "Subagent deadline expired while queued"
|
||||
result.completed_at = datetime.now()
|
||||
result.execution_finished.set()
|
||||
if result.cleanup_requested:
|
||||
cleanup_background_task(task_id)
|
||||
|
||||
try:
|
||||
_get_scheduler().submit(get_effective_user_id(), run_task, timeout=self.config.timeout_seconds,
|
||||
cancelled=result.cancel_event.is_set, expired=expired)
|
||||
except QueueFullError as exc:
|
||||
with _background_tasks_lock:
|
||||
result.status = SubagentStatus.FAILED
|
||||
result.error = str(exc)
|
||||
result.completed_at = datetime.now()
|
||||
result.execution_finished.set()
|
||||
return task_id
|
||||
|
||||
|
||||
@@ -787,7 +885,8 @@ def cleanup_background_task(task_id: str) -> None:
|
||||
SubagentStatus.CANCELLED,
|
||||
SubagentStatus.TIMED_OUT,
|
||||
}
|
||||
if is_terminal_status or result.completed_at is not None:
|
||||
result.cleanup_requested = True
|
||||
if (is_terminal_status or result.completed_at is not None) and result.execution_finished.is_set():
|
||||
del _background_tasks[task_id]
|
||||
logger.debug("Cleaned up background task: %s", task_id)
|
||||
else:
|
||||
|
||||
Reference in new issue
Block a user