sync update from LAN repo (source dhk-1 @ 3e87be55)

This commit is contained in:
gozeros committed 2026-10-21 15:06:08 +08:00
1 parent ab6068a7c1
commit 42f2a26fa8
1482 files changed
+227759 -9584

No files matched your search

@@ -0,0 +1,50 @@
"""Shared detection for adaptive context recovery (no assumed model window)."""
from collections.abc import Mapping
CONTEXT_RECOVERY_MESSAGE = "请补充本次需要处理的具体问题或关键材料,我会继续为你解答。"
_PATTERNS = (
"context_length_exceeded",
"maximum context length",
"max context length",
"context window exceeds limit",
"context window exceeded",
"exceeds the context window",
"max_model_len",
"prompt is too long",
"input is too long",
"too many tokens",
"input token count",
"prompt token count",
"exceeds the model's context",
"exceeds model context",
"上下文长度",
"最大上下文",
"输入 token 数超过",
"输入token数超过",
)
def is_context_length_error(exc: BaseException) -> bool:
"""Inspect provider error codes as well as text, without matching all 400s."""
values = [str(exc), str(getattr(exc, "code", "")), str(getattr(exc, "error_code", ""))]
# Compatible gateways may use numeric codes and put the only context-size
# signal in their message. Our adapter keeps that text private so __str__
# and framework logs cannot expose provider bodies; classify it in-process.
classification_detail = getattr(exc, "_classification_detail", None)
if isinstance(classification_detail, str):
values.append(classification_detail)
body = getattr(exc, "body", None)
if isinstance(body, Mapping):
error = body.get("error", body)
if isinstance(error, Mapping):
values.extend(str(error.get(key, "")) for key in ("code", "type", "message"))
return any(pattern in value.lower() for value in values for pattern in _PATTERNS)
class ContextRecoveryExhausted(RuntimeError):
"""Internal failure for contracts that cannot degrade to conversational text."""
def __init__(self) -> None:
super().__init__("context_length_exceeded: adaptive recovery exhausted")
@@ -2,8 +2,10 @@ import logging
from langchain.chat_models import BaseChatModel
from deerflow.config import get_app_config
from deerflow.config.app_config import AppConfig
from deerflow.config import get_app_config
from deerflow.config.app_config import AppConfig
from deerflow.config.model_config import WriteFileTransport
from deerflow.config.system_settings import get_write_file_compatibility_settings
from deerflow.reflection import resolve_class
from deerflow.tracing import build_tracing_callbacks
@@ -31,6 +33,96 @@ def _vllm_disable_chat_template_kwargs(chat_template_kwargs: dict) -> dict:
return disable_kwargs
# Qwen3.8 chat templates only accept these three effort names.
_QWEN_REASONING_EFFORT = {
"minimal": "low",
"low": "low",
"medium": "medium",
"high": "xhigh",
"xhigh": "xhigh",
}
_OPENAI_THINKING_OFF_EXTRA_BODY = {
"enable_thinking": False,
"chat_template_kwargs": {"enable_thinking": False},
"thinking": {"type": "disabled"},
}
def _is_vllm_provider(model_use: str) -> bool:
lowered = (model_use or "").lower()
return "vllm_provider" in lowered or lowered.endswith(":vllmchatmodel")
def _is_qwen38_model(model_config) -> bool:
blob = f"{getattr(model_config, 'name', '')} {getattr(model_config, 'model', '')}".lower()
return "qwen3.8" in blob
def _should_control_openai_thinking(model_config) -> bool:
"""Whether this OpenAI-compatible model needs explicit thinking on/off flags.
Qwen3.8 (and current vLLM reasoning parsers) default to thinking at ``xhigh``.
If DeerFlow does not send ``enable_thinking: false``, the first visible answer
token waits until that hidden reasoning finishes — third-party chat UIs that
disable thinking or stream the reasoning field therefore feel much faster.
"""
if getattr(model_config, "supports_thinking", False):
return True
if _is_vllm_provider(getattr(model_config, "use", "")):
return True
return _is_qwen38_model(model_config)
def _map_qwen_reasoning_effort(effort: object) -> str | None:
if effort is None:
return None
return _QWEN_REASONING_EFFORT.get(str(effort).strip().lower())
def _apply_openai_thinking_control(
model_config,
model_class,
model_settings_from_config: dict,
*,
thinking_enabled: bool,
reasoning_effort: object,
) -> None:
"""Inject vLLM/Qwen thinking flags when the UI toggle is on or off."""
if "extra_body" not in getattr(model_class, "model_fields", {}):
return
qwen_vllm = _is_vllm_provider(getattr(model_config, "use", "")) or _is_qwen38_model(model_config)
if thinking_enabled:
if not qwen_vllm:
return
enable_body: dict = {
"enable_thinking": True,
"chat_template_kwargs": {"enable_thinking": True},
}
mapped_effort = _map_qwen_reasoning_effort(reasoning_effort)
if mapped_effort:
enable_body["chat_template_kwargs"]["reasoning_effort"] = mapped_effort
model_settings_from_config["extra_body"] = _deep_merge_dicts(
model_settings_from_config.get("extra_body"),
enable_body,
)
return
if not _should_control_openai_thinking(model_config):
return
disable_body = (
{
"enable_thinking": False,
"chat_template_kwargs": {"enable_thinking": False},
}
if qwen_vllm
else _OPENAI_THINKING_OFF_EXTRA_BODY
)
model_settings_from_config["extra_body"] = _deep_merge_dicts(
model_settings_from_config.get("extra_body"),
disable_body,
)
def _enable_stream_usage_by_default(model_use_path: str, model_settings_from_config: dict) -> None:
"""Enable stream usage for OpenAI-compatible models unless explicitly configured.
@@ -80,8 +172,8 @@ def create_chat_model(name: str | None = None, thinking_enabled: bool = False, *
from deerflow.models.patched_openai import PatchedChatOpenAI
model_class = PatchedChatOpenAI
model_settings_from_config = model_config.model_dump(
exclude_none=True,
model_settings_from_config = model_config.model_dump(
exclude_none=True,
exclude={
"use",
"name",
@@ -92,9 +184,50 @@ def create_chat_model(name: str | None = None, thinking_enabled: bool = False, *
"when_thinking_enabled",
"when_thinking_disabled",
"thinking",
"supports_vision",
},
)
"supports_vision",
"write_file_transport",
},
)
configured_write_file_transport = model_config.write_file_transport
runtime_write_file_compatibility = get_write_file_compatibility_settings()
effective_write_file_transport = (
WriteFileTransport.BUFFERED_GATEWAY
if runtime_write_file_compatibility.enabled
else configured_write_file_transport
)
if effective_write_file_transport in {
WriteFileTransport.BUFFERED_GATEWAY,
WriteFileTransport.ARTIFACT_RENDERER,
WriteFileTransport.TEXT_BRIDGE,
}:
from deerflow.models.patched_openai import PatchedChatOpenAI
if not issubclass(model_class, PatchedChatOpenAI):
raise ValueError(
"write_file_transport=buffered_gateway/artifact_renderer/text_bridge is supported only by "
"langchain_openai:ChatOpenAI or PatchedChatOpenAI models"
)
if model_config.use_responses_api:
raise ValueError(
"write_file_transport=buffered_gateway/artifact_renderer/text_bridge requires the "
"OpenAI-compatible chat-completions API, not use_responses_api"
)
# The admin runtime switch overrides every compatible model on its next
# construction. With the switch off, retain the legacy per-model value.
model_settings_from_config["write_file_transport"] = effective_write_file_transport.value
# ``max_input_tokens`` is a DeerFlow-side safety limit, not an OpenAI API
# request parameter. ModelConfig intentionally allows provider-specific
# extras, so without removing it here ChatOpenAI would forward the unknown
# key to some compatible gateways and turn the protection itself into a
# HTTP 400. Mirror it into LangChain's local-only ``profile`` metadata
# when the selected model supports that field; the final context guard
# reads the original model config and therefore also works for adapters
# without profiles.
configured_max_input_tokens = model_settings_from_config.pop("max_input_tokens", None)
if configured_max_input_tokens is not None and "profile" in getattr(model_class, "model_fields", {}):
profile = dict(model_settings_from_config.get("profile") or {})
profile["max_input_tokens"] = configured_max_input_tokens
model_settings_from_config["profile"] = profile
# Compute effective when_thinking_enabled by merging in the `thinking` shortcut field.
# The `thinking` shortcut is equivalent to setting when_thinking_enabled["thinking"].
has_thinking_settings = (model_config.when_thinking_enabled is not None) or (model_config.thinking is not None)
@@ -102,11 +235,18 @@ def create_chat_model(name: str | None = None, thinking_enabled: bool = False, *
if model_config.thinking is not None:
merged_thinking = {**(effective_wte.get("thinking") or {}), **model_config.thinking}
effective_wte = {**effective_wte, "thinking": merged_thinking}
if thinking_enabled and has_thinking_settings:
if not model_config.supports_thinking:
raise ValueError(f"Model {name} does not support thinking. Set `supports_thinking` to true in the `config.yaml` to enable thinking.") from None
if effective_wte:
model_settings_from_config.update(effective_wte)
if thinking_enabled and has_thinking_settings:
if not model_config.supports_thinking:
raise ValueError(f"Model {name} does not support thinking. Set `supports_thinking` to true in the `config.yaml` to enable thinking.") from None
if effective_wte:
model_settings_from_config.update(effective_wte)
elif thinking_enabled and model_config.supports_thinking and model_config.use == "langchain_openai:ChatOpenAI":
# vLLM/Qwen OpenAI-compatible endpoints commonly require this explicit
# request flag before they emit delta.reasoning.
model_settings_from_config["extra_body"] = _deep_merge_dicts(
model_settings_from_config.get("extra_body"),
{"enable_thinking": True},
)
if not thinking_enabled:
if model_config.when_thinking_disabled is not None:
# User-provided disable settings take full precedence
@@ -127,10 +267,19 @@ def create_chat_model(name: str | None = None, thinking_enabled: bool = False, *
elif has_thinking_settings and effective_wte.get("thinking", {}).get("type"):
# Native langchain_anthropic: thinking is a direct constructor parameter
model_settings_from_config["thinking"] = {"type": "disabled"}
requested_reasoning_effort = kwargs.get("reasoning_effort")
if not model_config.supports_reasoning_effort:
kwargs.pop("reasoning_effort", None)
model_settings_from_config.pop("reasoning_effort", None)
_apply_openai_thinking_control(
model_config,
model_class,
model_settings_from_config,
thinking_enabled=thinking_enabled,
reasoning_effort=requested_reasoning_effort,
)
# Hard override: force-disable thinking regardless of the per-model config shape.
# Injected only for OpenAI-compatible classes (those declaring `extra_body`);
# other providers would reject the kwarg, so they are skipped (they ignore it).
@@ -139,15 +288,15 @@ def create_chat_model(name: str | None = None, thinking_enabled: bool = False, *
# (DeepSeek-V3.1+/Qwen3/GLM hybrid models on api.siliconflow.cn — the knob the
# nested ``chat_template_kwargs`` shape does NOT reach), vLLM/Qwen native
# ``chat_template_kwargs.enable_thinking``, and Zhipu GLM ``thinking.type``.
# Only models that explicitly opt into thinking should receive vendor-specific
# disable payloads. Generic OpenAI-compatible gateways (including LiteLLM)
# may reject these otherwise-unknown fields instead of forwarding them.
if (
force_disable_thinking
and not thinking_enabled
and model_config.supports_thinking
and "extra_body" in getattr(model_class, "model_fields", {})
):
# Only models that explicitly opt into thinking should receive vendor-specific
# disable payloads. Generic OpenAI-compatible gateways (including LiteLLM)
# may reject these otherwise-unknown fields instead of forwarding them.
if (
force_disable_thinking
and not thinking_enabled
and model_config.supports_thinking
and "extra_body" in getattr(model_class, "model_fields", {})
):
model_settings_from_config["extra_body"] = _deep_merge_dicts(
model_settings_from_config.get("extra_body"),
{"enable_thinking": False, "chat_template_kwargs": {"enable_thinking": False}, "thinking": {"type": "disabled"}},
@@ -186,11 +335,26 @@ def create_chat_model(name: str | None = None, thinking_enabled: bool = False, *
if "stream_usage" in getattr(model_class, "model_fields", {}):
model_settings_from_config["stream_usage"] = True
model_instance = model_class(**kwargs, **model_settings_from_config)
callbacks = build_tracing_callbacks()
if callbacks:
existing_callbacks = model_instance.callbacks or []
model_instance = model_class(**kwargs, **model_settings_from_config)
# Always emit a compact model lifecycle trace. In particular, the first
# streamed callback proves that the upstream endpoint accepted the request
# and started returning data, including reasoning-only chunks whose normal
# content string is empty.
from deerflow.models.timing_callback import ModelTimingCallback
existing_callbacks = model_instance.callbacks or []
model_instance.callbacks = [
*existing_callbacks,
ModelTimingCallback(
model_name=name,
base_url=str(model_config.base_url) if model_config.base_url else None,
),
]
callbacks = build_tracing_callbacks()
if callbacks:
existing_callbacks = model_instance.callbacks or []
model_instance.callbacks = [*existing_callbacks, *callbacks]
logger.debug(f"Tracing attached to model '{name}' with providers={len(callbacks)}")
return model_instance
@@ -6,7 +6,6 @@ from typing import Any
from langchain_core.messages import BaseMessage, SystemMessage
_SYSTEM_SEPARATOR = "\n\n---\n\n"
@@ -35,7 +34,11 @@ def normalize_langchain_messages(messages: list[BaseMessage]) -> list[BaseMessag
return messages
first_non_system_index = next((idx for idx, message in enumerate(messages) if not isinstance(message, SystemMessage)), len(messages))
if len(system_messages) == first_non_system_index:
# A number of strict chat templates accept *one* system message, and only
# at index zero. Two adjacent leading system messages are therefore not a
# valid no-op case: LangChain's agent prompt followed by DeerFlow's
# transient compaction summary must still be folded into one message.
if len(system_messages) == 1 and first_non_system_index == 1:
return messages
merged = system_messages[0].model_copy(
@@ -46,20 +49,38 @@ def normalize_langchain_messages(messages: list[BaseMessage]) -> list[BaseMessag
return [merged, *(message for message in messages if not isinstance(message, SystemMessage))]
def normalize_model_request_messages(
system_message: SystemMessage | None,
messages: list[BaseMessage],
) -> tuple[SystemMessage | None, list[BaseMessage]]:
"""Normalize LangChain's split ``ModelRequest`` message representation.
``ModelRequest.messages`` is documented as excluding the system prompt.
Transient middleware (notably context compaction) can nevertheless place a
``SystemMessage`` in that list. Fold every such message into the dedicated
system slot so adapters without a DeerFlow-specific payload patch also
receive exactly one leading system message.
"""
if system_message is None and not any(isinstance(message, SystemMessage) for message in messages):
return None, messages
combined = [system_message, *messages] if system_message is not None else messages
normalized = normalize_langchain_messages(combined)
if normalized and isinstance(normalized[0], SystemMessage):
return normalized[0], normalized[1:]
return None, normalized
def _normalize_payload_message_list(messages: list[Any]) -> list[Any]:
system_messages = [message for message in messages if isinstance(message, dict) and message.get("role") == "system"]
if not system_messages:
return messages
first_non_system_index = next(
(
idx
for idx, message in enumerate(messages)
if not (isinstance(message, dict) and message.get("role") == "system")
),
(idx for idx, message in enumerate(messages) if not (isinstance(message, dict) and message.get("role") == "system")),
len(messages),
)
if len(system_messages) == first_non_system_index:
if len(system_messages) == 1 and first_non_system_index == 1:
return messages
merged_system = dict(system_messages[0])
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,140 @@
"""Normalize OpenAI-compatible thinking fields across provider versions.
Older vLLM / DeepSeek put thinking in ``reasoning_content``. Newer vLLM
reasoning parsers put it in ``reasoning``. DeerFlow's UI and downstream code
canonicalize on ``AIMessage.additional_kwargs.reasoning_content``.
Only fields that actually appeared on a response are echoed back on later
turns, so models that still use ``reasoning_content`` (or send neither) are
unchanged.
"""
from __future__ import annotations
import json
from typing import Any
from langchain_core.messages import AIMessage, AIMessageChunk
def reasoning_to_text(reasoning: Any) -> str:
"""Best-effort extraction of readable reasoning text from provider payloads."""
if isinstance(reasoning, str):
return reasoning
if isinstance(reasoning, list):
parts = [reasoning_to_text(item) for item in reasoning]
return "".join(part for part in parts if part)
if isinstance(reasoning, dict):
for key in ("text", "content", "reasoning"):
value = reasoning.get(key)
if isinstance(value, str):
return value
if value is not None:
text = reasoning_to_text(value)
if text:
return text
try:
return json.dumps(reasoning, ensure_ascii=False)
except TypeError:
return str(reasoning)
if reasoning is None:
return ""
try:
return json.dumps(reasoning, ensure_ascii=False)
except TypeError:
return str(reasoning)
def extract_reasoning_text(payload: dict | None) -> str | None:
"""Read thinking text from ``reasoning_content`` or ``reasoning``.
Prefer ``reasoning_content`` when both are present so older DeepSeek /
vLLM payloads keep their original string. Newer vLLM that only sends
``reasoning`` (string or structured) is still captured.
"""
if not isinstance(payload, dict):
return None
for key in ("reasoning_content", "reasoning"):
value = payload.get(key)
if value is None:
continue
text = reasoning_to_text(value)
if text:
return text
return None
def _merge_streamed_value(existing: Any, incoming: Any) -> Any:
if isinstance(existing, str) and isinstance(incoming, str):
if incoming and incoming not in existing:
return f"{existing}{incoming}"
return existing or incoming
return incoming if incoming is not None else existing
def apply_reasoning_from_payload(
message: AIMessage | AIMessageChunk,
payload: dict | None,
*,
append: bool = True,
) -> AIMessage | AIMessageChunk:
"""Copy provider thinking fields onto ``additional_kwargs`` for UI + echo-back."""
if not isinstance(payload, dict):
return message
additional_kwargs = dict(message.additional_kwargs)
changed = False
if "reasoning" in payload and payload.get("reasoning") is not None:
incoming = payload.get("reasoning")
existing = additional_kwargs.get("reasoning")
merged = _merge_streamed_value(existing, incoming) if append else incoming
if merged is not None and merged != existing:
additional_kwargs["reasoning"] = merged
changed = True
text = extract_reasoning_text(payload)
if text:
existing = additional_kwargs.get("reasoning_content")
if append and isinstance(existing, str) and existing:
if text not in existing:
additional_kwargs["reasoning_content"] = f"{existing}{text}"
changed = True
elif text != existing:
additional_kwargs["reasoning_content"] = text
changed = True
if not changed:
return message
return message.model_copy(update={"additional_kwargs": additional_kwargs})
def fill_ui_reasoning_content(message: AIMessage | AIMessageChunk) -> AIMessage | AIMessageChunk:
"""If LangChain stored ``reasoning`` but not the UI field, copy the text across."""
kwargs = message.additional_kwargs
reasoning = kwargs.get("reasoning")
if reasoning is None or kwargs.get("reasoning_content"):
return message
return apply_reasoning_from_payload(message, {"reasoning": reasoning}, append=False)
def restore_reasoning_fields(payload_msg: dict, orig_msg: AIMessage) -> None:
"""Re-inject the thinking field this assistant turn originally carried.
Newer vLLM stores thinking on ``reasoning`` and we also copy a UI-facing
``reasoning_content`` string from it. Echo the wire field that actually
arrived: ``reasoning`` wins when present so we do not send a synthesized
``reasoning_content`` to a parser that no longer accepts it. Older
DeepSeek / vLLM turns only have ``reasoning_content`` and stay on that
field.
"""
reasoning = orig_msg.additional_kwargs.get("reasoning")
if reasoning is not None:
payload_msg["reasoning"] = reasoning
return
reasoning_content = orig_msg.additional_kwargs.get("reasoning_content")
if reasoning_content is not None:
payload_msg["reasoning_content"] = reasoning_content
@@ -0,0 +1,260 @@
"""Concise model connection and latency diagnostics for production logs."""
from __future__ import annotations
import asyncio
import atexit
import logging
import queue
import threading
import time
from dataclasses import dataclass
from typing import Any
from urllib.parse import urlsplit
from uuid import UUID
from langchain_core.callbacks import BaseCallbackHandler
logger = logging.getLogger(__name__)
class _TimingLogDispatcher:
"""Keep slow log handlers off both the event loop and its tool executor.
Only already-sanitized, small lifecycle records enter this bounded queue.
A daemon thread invokes the existing handlers; producers never wait for
their I/O. Full queues drop diagnostics and report the count when the sink
recovers, instead of retaining unbounded records or stalling generation.
"""
def __init__(self, *, capacity: int = 4096) -> None:
self._queue: queue.Queue[logging.LogRecord] = queue.Queue(maxsize=capacity)
self._lock = threading.Lock()
self._thread: threading.Thread | None = None
self._closed = False
self._dropped = 0
def emit(self, level: int, message: str, *args: Any) -> None:
if not logger.isEnabledFor(level):
return
# Preserve the event's wall-clock timestamp even if delivery is delayed.
# Formatting, filters and handler I/O run only on the consumer thread.
record = logger.makeRecord(logger.name, level, __file__, 0, message, args, None)
with self._lock:
if self._closed:
return
try:
self._queue.put_nowait(record)
except queue.Full:
self._dropped += 1
return
if self._thread is None:
self._thread = threading.Thread(target=self._run, name="deerflow-model-timing-log", daemon=True)
self._thread.start()
def _run(self) -> None:
while True:
try:
record = self._queue.get(timeout=0.1)
except queue.Empty:
with self._lock:
if self._closed:
return
continue
try:
logger.handle(record)
except Exception:
# Diagnostic sinks must not fail a model call or kill the worker.
pass
finally:
self._queue.task_done()
with self._lock:
dropped, self._dropped = self._dropped, 0
if dropped:
try:
logger.warning("LLM_TIMING_LOG_DROPPED count=%d reason=queue_full", dropped)
except Exception:
pass
def close(self, timeout: float = 0.5) -> None:
"""Best-effort drain without making process exit wait on a log sink."""
with self._lock:
self._closed = True
thread = self._thread
if thread is not None and thread is not threading.current_thread():
thread.join(timeout=timeout)
_timing_logs = _TimingLogDispatcher()
atexit.register(_timing_logs.close)
def _safe_base_url(value: str | None) -> str:
"""Log only the upstream origin, never credentials, query, or URL path."""
if not value:
return "-"
try:
parsed = urlsplit(value)
except (TypeError, ValueError):
return "-"
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
return "-"
host = parsed.hostname
try:
port = parsed.port
except ValueError:
return "-"
return f"{parsed.scheme}://{host}:{port}" if port is not None else f"{parsed.scheme}://{host}"
@dataclass(slots=True)
class _CallTiming:
started: float
first_chunk: float | None = None
class ModelTimingCallback(BaseCallbackHandler):
"""Optionally record timing inline and deliver logs on a dedicated queue."""
# LangChain otherwise sends each synchronous callback, including every
# token's no-op after the first, through the shared tool thread pool.
# When enabled, inline work is limited to timestamps, short locked state
# changes and a non-blocking enqueue, never synchronous handler I/O.
run_inline = False
def __init__(self, *, model_name: str, base_url: str | None) -> None:
from deerflow.config.runtime_performance_config import get_runtime_performance_config
self.run_inline = get_runtime_performance_config().model_timing_nonblocking_enabled
self._model_name = model_name
self._base_url = _safe_base_url(base_url)
self._calls: dict[str, _CallTiming] = {}
self._lock = threading.Lock()
def _log(self, level: int, message: str, *args: Any) -> None:
if self.run_inline:
_timing_logs.emit(level, message, *args)
else:
# The default-off path retains LangChain's prior callback dispatch
# and synchronous logging behavior for deployment compatibility.
logger.log(level, message, *args)
def on_chat_model_start(
self,
serialized: dict[str, Any],
messages: list[list[Any]],
*,
run_id: UUID,
**kwargs: Any,
) -> None:
rid = str(run_id)
now = time.monotonic()
with self._lock:
self._calls[rid] = _CallTiming(started=now)
self._log(
logging.INFO,
"LLM_CONNECT_START model=%s base_url=%s run_id=%s message_batches=%d message_count=%d",
self._model_name,
self._base_url,
rid,
len(messages),
sum(len(batch) for batch in messages),
)
def on_llm_new_token(
self,
token: str,
*,
run_id: UUID,
chunk: Any | None = None,
**kwargs: Any,
) -> None:
"""Treat the first callback as connection success, including reasoning-only chunks."""
rid = str(run_id)
now = time.monotonic()
with self._lock:
timing = self._calls.get(rid)
if timing is None or timing.first_chunk is not None:
return
timing.first_chunk = now
first_chunk_ms = int((now - timing.started) * 1000)
self._log(
logging.INFO,
"LLM_CONNECT_OK model=%s base_url=%s run_id=%s first_chunk_ms=%d",
self._model_name,
self._base_url,
rid,
first_chunk_ms,
)
def on_llm_end(self, response: Any, *, run_id: UUID, **kwargs: Any) -> None:
self._finish(run_id=run_id, error=None)
def on_llm_error(self, error: BaseException, *, run_id: UUID, **kwargs: Any) -> None:
self._finish(run_id=run_id, error=error)
def _finish(self, *, run_id: UUID, error: BaseException | None) -> None:
rid = str(run_id)
now = time.monotonic()
cancelled = isinstance(error, (asyncio.CancelledError, GeneratorExit))
with self._lock:
timing = self._calls.pop(rid, None)
if timing is None:
if cancelled:
self._log(
logging.INFO,
"LLM_REQUEST_CANCELLED model=%s base_url=%s run_id=%s status=cancelled error_type=%s",
self._model_name,
self._base_url,
rid,
type(error).__name__,
)
return
self._log(
logging.ERROR,
"LLM_REQUEST_FAILED model=%s base_url=%s run_id=%s error_type=%s",
self._model_name,
self._base_url,
rid,
type(error).__name__ if error else "UnknownLifecycle",
)
return
total_ms = int((now - timing.started) * 1000)
first_chunk_ms = int((timing.first_chunk - timing.started) * 1000) if timing.first_chunk else None
if error is None:
self._log(
logging.INFO,
"LLM_REQUEST_DONE model=%s base_url=%s run_id=%s status=success first_chunk_ms=%s total_ms=%d",
self._model_name,
self._base_url,
rid,
first_chunk_ms if first_chunk_ms is not None else "none",
total_ms,
)
return
if cancelled:
self._log(
logging.INFO,
"LLM_REQUEST_CANCELLED model=%s base_url=%s run_id=%s status=cancelled first_chunk_ms=%s total_ms=%d error_type=%s",
self._model_name,
self._base_url,
rid,
first_chunk_ms if first_chunk_ms is not None else "none",
total_ms,
type(error).__name__,
)
return
self._log(
logging.ERROR,
"LLM_REQUEST_FAILED model=%s base_url=%s run_id=%s status=error first_chunk_ms=%s total_ms=%d error_type=%s",
self._model_name,
self._base_url,
rid,
first_chunk_ms if first_chunk_ms is not None else "none",
total_ms,
type(error).__name__,
)
@@ -6,15 +6,16 @@ from assistant messages and streaming deltas. That breaks interleaved
thinking/tool-call flows because vLLM expects the assistant's prior reasoning to
be echoed back on subsequent turns.
This provider preserves ``reasoning`` on:
- non-streaming responses
- streaming deltas
- multi-turn request payloads
This provider preserves ``reasoning`` (newer vLLM) and
``reasoning_content`` (older vLLM / DeepSeek-style parsers) on:
- non-streaming responses
- streaming deltas
- multi-turn request payloads
UI code still reads the canonical ``additional_kwargs.reasoning_content``.
"""
from __future__ import annotations
import json
from collections.abc import Mapping
from typing import Any, cast
@@ -32,11 +33,12 @@ from langchain_core.messages import (
)
from langchain_core.messages.tool import tool_call_chunk
from langchain_core.outputs import ChatGeneration, ChatGenerationChunk, ChatResult
from langchain_openai import ChatOpenAI
from langchain_openai.chat_models.base import _create_usage_metadata
from deerflow.models.message_ordering import normalize_chat_payload_messages
from langchain_openai import ChatOpenAI
from langchain_openai.chat_models.base import _create_usage_metadata
from deerflow.models.message_ordering import normalize_chat_payload_messages
from deerflow.models.reasoning_fields import apply_reasoning_from_payload, restore_reasoning_fields
def _normalize_vllm_chat_template_kwargs(payload: dict[str, Any]) -> None:
"""Map DeerFlow's legacy ``thinking`` toggle to vLLM/Qwen's ``enable_thinking``.
@@ -66,31 +68,9 @@ def _normalize_vllm_chat_template_kwargs(payload: dict[str, Any]) -> None:
def _reasoning_to_text(reasoning: Any) -> str:
"""Best-effort extraction of readable reasoning text from vLLM payloads."""
if isinstance(reasoning, str):
return reasoning
from deerflow.models.reasoning_fields import reasoning_to_text
if isinstance(reasoning, list):
parts = [_reasoning_to_text(item) for item in reasoning]
return "".join(part for part in parts if part)
if isinstance(reasoning, dict):
for key in ("text", "content", "reasoning"):
value = reasoning.get(key)
if isinstance(value, str):
return value
if value is not None:
text = _reasoning_to_text(value)
if text:
return text
try:
return json.dumps(reasoning, ensure_ascii=False)
except TypeError:
return str(reasoning)
try:
return json.dumps(reasoning, ensure_ascii=False)
except TypeError:
return str(reasoning)
return reasoning_to_text(reasoning)
def _convert_delta_to_message_chunk_with_reasoning(_dict: Mapping[str, Any], default_class: type[BaseMessageChunk]) -> BaseMessageChunk:
@@ -106,12 +86,8 @@ def _convert_delta_to_message_chunk_with_reasoning(_dict: Mapping[str, Any], def
function_call["name"] = ""
additional_kwargs["function_call"] = function_call
reasoning = _dict.get("reasoning")
if reasoning is not None:
additional_kwargs["reasoning"] = reasoning
reasoning_text = _reasoning_to_text(reasoning)
if reasoning_text:
additional_kwargs["reasoning_content"] = reasoning_text
seed = AIMessageChunk(content="", additional_kwargs=additional_kwargs)
additional_kwargs = dict(apply_reasoning_from_payload(seed, dict(_dict)).additional_kwargs)
tool_call_chunks = []
if raw_tool_calls := _dict.get("tool_calls"):
@@ -151,11 +127,7 @@ def _convert_delta_to_message_chunk_with_reasoning(_dict: Mapping[str, Any], def
def _restore_reasoning_field(payload_msg: dict[str, Any], orig_msg: AIMessage) -> None:
"""Re-inject vLLM reasoning onto outgoing assistant messages."""
reasoning = orig_msg.additional_kwargs.get("reasoning")
if reasoning is None:
reasoning = orig_msg.additional_kwargs.get("reasoning_content")
if reasoning is not None:
payload_msg["reasoning"] = reasoning
restore_reasoning_fields(payload_msg, orig_msg)
class VllmChatModel(ChatOpenAI):
@@ -184,14 +156,14 @@ class VllmChatModel(ChatOpenAI):
for payload_msg, orig_msg in zip(payload_messages, original_messages):
if payload_msg.get("role") == "assistant" and isinstance(orig_msg, AIMessage):
_restore_reasoning_field(payload_msg, orig_msg)
else:
ai_messages = [message for message in original_messages if isinstance(message, AIMessage)]
assistant_payloads = [message for message in payload_messages if message.get("role") == "assistant"]
for payload_msg, ai_msg in zip(assistant_payloads, ai_messages):
_restore_reasoning_field(payload_msg, ai_msg)
normalize_chat_payload_messages(payload)
return payload
else:
ai_messages = [message for message in original_messages if isinstance(message, AIMessage)]
assistant_payloads = [message for message in payload_messages if message.get("role") == "assistant"]
for payload_msg, ai_msg in zip(assistant_payloads, ai_messages):
_restore_reasoning_field(payload_msg, ai_msg)
normalize_chat_payload_messages(payload)
return payload
def _create_chat_result(self, response: dict | openai.BaseModel, generation_info: dict | None = None) -> ChatResult:
"""Preserve vLLM reasoning on non-streaming responses."""
@@ -204,13 +176,12 @@ class VllmChatModel(ChatOpenAI):
message = generation.message
if not isinstance(message, AIMessage):
continue
reasoning = choice.get("message", {}).get("reasoning")
if reasoning is None:
continue
message.additional_kwargs["reasoning"] = reasoning
reasoning_text = _reasoning_to_text(reasoning)
if reasoning_text:
message.additional_kwargs["reasoning_content"] = reasoning_text
choice_message = choice.get("message", {}) if isinstance(choice, dict) else {}
generation.message = apply_reasoning_from_payload(
message,
choice_message if isinstance(choice_message, dict) else None,
append=False,
)
return result