sync update from LAN repo (source ji @ 85078a31)

This commit is contained in:
gozeros committed 2026-10-21 15:09:16 +08:00
1 parent 5d6627af50
commit e602d88695
864 files changed
+99150 -6346

No files matched your search

@@ -0,0 +1,44 @@
"""Shared detection for adaptive context recovery (no assumed model window)."""
from collections.abc import Mapping
CONTEXT_RECOVERY_MESSAGE = "请补充本次需要处理的具体问题或关键材料,我会继续为你解答。"
_PATTERNS = (
"context_length_exceeded",
"maximum context length",
"max context length",
"context window exceeds limit",
"context window exceeded",
"exceeds the context window",
"max_model_len",
"prompt is too long",
"input is too long",
"too many tokens",
"input token count",
"prompt token count",
"exceeds the model's context",
"exceeds model context",
"上下文长度",
"最大上下文",
"输入 token 数超过",
"输入token数超过",
)
def is_context_length_error(exc: BaseException) -> bool:
"""Inspect provider error codes as well as text, without matching all 400s."""
values = [str(exc), str(getattr(exc, "code", "")), str(getattr(exc, "error_code", ""))]
body = getattr(exc, "body", None)
if isinstance(body, Mapping):
error = body.get("error", body)
if isinstance(error, Mapping):
values.extend(str(error.get(key, "")) for key in ("code", "type", "message"))
return any(pattern in value.lower() for value in values for pattern in _PATTERNS)
class ContextRecoveryExhausted(RuntimeError):
"""Internal failure for contracts that cannot degrade to conversational text."""
def __init__(self) -> None:
super().__init__("context_length_exceeded: adaptive recovery exhausted")
@@ -31,6 +31,96 @@ def _vllm_disable_chat_template_kwargs(chat_template_kwargs: dict) -> dict:
return disable_kwargs
# Qwen3.8 chat templates only accept these three effort names.
_QWEN_REASONING_EFFORT = {
"minimal": "low",
"low": "low",
"medium": "medium",
"high": "xhigh",
"xhigh": "xhigh",
}
_OPENAI_THINKING_OFF_EXTRA_BODY = {
"enable_thinking": False,
"chat_template_kwargs": {"enable_thinking": False},
"thinking": {"type": "disabled"},
}
def _is_vllm_provider(model_use: str) -> bool:
lowered = (model_use or "").lower()
return "vllm_provider" in lowered or lowered.endswith(":vllmchatmodel")
def _is_qwen38_model(model_config) -> bool:
blob = f"{getattr(model_config, 'name', '')} {getattr(model_config, 'model', '')}".lower()
return "qwen3.8" in blob
def _should_control_openai_thinking(model_config) -> bool:
"""Whether this OpenAI-compatible model needs explicit thinking on/off flags.
Qwen3.8 (and current vLLM reasoning parsers) default to thinking at ``xhigh``.
If DeerFlow does not send ``enable_thinking: false``, the first visible answer
token waits until that hidden reasoning finishes — third-party chat UIs that
disable thinking or stream the reasoning field therefore feel much faster.
"""
if getattr(model_config, "supports_thinking", False):
return True
if _is_vllm_provider(getattr(model_config, "use", "")):
return True
return _is_qwen38_model(model_config)
def _map_qwen_reasoning_effort(effort: object) -> str | None:
if effort is None:
return None
return _QWEN_REASONING_EFFORT.get(str(effort).strip().lower())
def _apply_openai_thinking_control(
model_config,
model_class,
model_settings_from_config: dict,
*,
thinking_enabled: bool,
reasoning_effort: object,
) -> None:
"""Inject vLLM/Qwen thinking flags when the UI toggle is on or off."""
if "extra_body" not in getattr(model_class, "model_fields", {}):
return
qwen_vllm = _is_vllm_provider(getattr(model_config, "use", "")) or _is_qwen38_model(model_config)
if thinking_enabled:
if not qwen_vllm:
return
enable_body: dict = {
"enable_thinking": True,
"chat_template_kwargs": {"enable_thinking": True},
}
mapped_effort = _map_qwen_reasoning_effort(reasoning_effort)
if mapped_effort:
enable_body["chat_template_kwargs"]["reasoning_effort"] = mapped_effort
model_settings_from_config["extra_body"] = _deep_merge_dicts(
model_settings_from_config.get("extra_body"),
enable_body,
)
return
if not _should_control_openai_thinking(model_config):
return
disable_body = (
{
"enable_thinking": False,
"chat_template_kwargs": {"enable_thinking": False},
}
if qwen_vllm
else _OPENAI_THINKING_OFF_EXTRA_BODY
)
model_settings_from_config["extra_body"] = _deep_merge_dicts(
model_settings_from_config.get("extra_body"),
disable_body,
)
def _enable_stream_usage_by_default(model_use_path: str, model_settings_from_config: dict) -> None:
"""Enable stream usage for OpenAI-compatible models unless explicitly configured.
@@ -80,8 +170,8 @@ def create_chat_model(name: str | None = None, thinking_enabled: bool = False, *
from deerflow.models.patched_openai import PatchedChatOpenAI
model_class = PatchedChatOpenAI
model_settings_from_config = model_config.model_dump(
exclude_none=True,
model_settings_from_config = model_config.model_dump(
exclude_none=True,
exclude={
"use",
"name",
@@ -92,9 +182,22 @@ def create_chat_model(name: str | None = None, thinking_enabled: bool = False, *
"when_thinking_enabled",
"when_thinking_disabled",
"thinking",
"supports_vision",
},
)
"supports_vision",
},
)
# ``max_input_tokens`` is a DeerFlow-side safety limit, not an OpenAI API
# request parameter. ModelConfig intentionally allows provider-specific
# extras, so without removing it here ChatOpenAI would forward the unknown
# key to some compatible gateways and turn the protection itself into a
# HTTP 400. Mirror it into LangChain's local-only ``profile`` metadata
# when the selected model supports that field; the final context guard
# reads the original model config and therefore also works for adapters
# without profiles.
configured_max_input_tokens = model_settings_from_config.pop("max_input_tokens", None)
if configured_max_input_tokens is not None and "profile" in getattr(model_class, "model_fields", {}):
profile = dict(model_settings_from_config.get("profile") or {})
profile["max_input_tokens"] = configured_max_input_tokens
model_settings_from_config["profile"] = profile
# Compute effective when_thinking_enabled by merging in the `thinking` shortcut field.
# The `thinking` shortcut is equivalent to setting when_thinking_enabled["thinking"].
has_thinking_settings = (model_config.when_thinking_enabled is not None) or (model_config.thinking is not None)
@@ -102,11 +205,18 @@ def create_chat_model(name: str | None = None, thinking_enabled: bool = False, *
if model_config.thinking is not None:
merged_thinking = {**(effective_wte.get("thinking") or {}), **model_config.thinking}
effective_wte = {**effective_wte, "thinking": merged_thinking}
if thinking_enabled and has_thinking_settings:
if not model_config.supports_thinking:
raise ValueError(f"Model {name} does not support thinking. Set `supports_thinking` to true in the `config.yaml` to enable thinking.") from None
if effective_wte:
model_settings_from_config.update(effective_wte)
if thinking_enabled and has_thinking_settings:
if not model_config.supports_thinking:
raise ValueError(f"Model {name} does not support thinking. Set `supports_thinking` to true in the `config.yaml` to enable thinking.") from None
if effective_wte:
model_settings_from_config.update(effective_wte)
elif thinking_enabled and model_config.supports_thinking and model_config.use == "langchain_openai:ChatOpenAI":
# vLLM/Qwen OpenAI-compatible endpoints commonly require this explicit
# request flag before they emit delta.reasoning.
model_settings_from_config["extra_body"] = _deep_merge_dicts(
model_settings_from_config.get("extra_body"),
{"enable_thinking": True},
)
if not thinking_enabled:
if model_config.when_thinking_disabled is not None:
# User-provided disable settings take full precedence
@@ -127,10 +237,19 @@ def create_chat_model(name: str | None = None, thinking_enabled: bool = False, *
elif has_thinking_settings and effective_wte.get("thinking", {}).get("type"):
# Native langchain_anthropic: thinking is a direct constructor parameter
model_settings_from_config["thinking"] = {"type": "disabled"}
requested_reasoning_effort = kwargs.get("reasoning_effort")
if not model_config.supports_reasoning_effort:
kwargs.pop("reasoning_effort", None)
model_settings_from_config.pop("reasoning_effort", None)
_apply_openai_thinking_control(
model_config,
model_class,
model_settings_from_config,
thinking_enabled=thinking_enabled,
reasoning_effort=requested_reasoning_effort,
)
# Hard override: force-disable thinking regardless of the per-model config shape.
# Injected only for OpenAI-compatible classes (those declaring `extra_body`);
# other providers would reject the kwarg, so they are skipped (they ignore it).
@@ -139,15 +258,15 @@ def create_chat_model(name: str | None = None, thinking_enabled: bool = False, *
# (DeepSeek-V3.1+/Qwen3/GLM hybrid models on api.siliconflow.cn — the knob the
# nested ``chat_template_kwargs`` shape does NOT reach), vLLM/Qwen native
# ``chat_template_kwargs.enable_thinking``, and Zhipu GLM ``thinking.type``.
# Only models that explicitly opt into thinking should receive vendor-specific
# disable payloads. Generic OpenAI-compatible gateways (including LiteLLM)
# may reject these otherwise-unknown fields instead of forwarding them.
if (
force_disable_thinking
and not thinking_enabled
and model_config.supports_thinking
and "extra_body" in getattr(model_class, "model_fields", {})
):
# Only models that explicitly opt into thinking should receive vendor-specific
# disable payloads. Generic OpenAI-compatible gateways (including LiteLLM)
# may reject these otherwise-unknown fields instead of forwarding them.
if (
force_disable_thinking
and not thinking_enabled
and model_config.supports_thinking
and "extra_body" in getattr(model_class, "model_fields", {})
):
model_settings_from_config["extra_body"] = _deep_merge_dicts(
model_settings_from_config.get("extra_body"),
{"enable_thinking": False, "chat_template_kwargs": {"enable_thinking": False}, "thinking": {"type": "disabled"}},
@@ -186,11 +305,26 @@ def create_chat_model(name: str | None = None, thinking_enabled: bool = False, *
if "stream_usage" in getattr(model_class, "model_fields", {}):
model_settings_from_config["stream_usage"] = True
model_instance = model_class(**kwargs, **model_settings_from_config)
callbacks = build_tracing_callbacks()
if callbacks:
existing_callbacks = model_instance.callbacks or []
model_instance = model_class(**kwargs, **model_settings_from_config)
# Always emit a compact model lifecycle trace. In particular, the first
# streamed callback proves that the upstream endpoint accepted the request
# and started returning data, including reasoning-only chunks whose normal
# content string is empty.
from deerflow.models.timing_callback import ModelTimingCallback
existing_callbacks = model_instance.callbacks or []
model_instance.callbacks = [
*existing_callbacks,
ModelTimingCallback(
model_name=name,
base_url=str(model_config.base_url) if model_config.base_url else None,
),
]
callbacks = build_tracing_callbacks()
if callbacks:
existing_callbacks = model_instance.callbacks or []
model_instance.callbacks = [*existing_callbacks, *callbacks]
logger.debug(f"Tracing attached to model '{name}' with providers={len(callbacks)}")
return model_instance
@@ -6,7 +6,6 @@ from typing import Any
from langchain_core.messages import BaseMessage, SystemMessage
_SYSTEM_SEPARATOR = "\n\n---\n\n"
@@ -35,7 +34,11 @@ def normalize_langchain_messages(messages: list[BaseMessage]) -> list[BaseMessag
return messages
first_non_system_index = next((idx for idx, message in enumerate(messages) if not isinstance(message, SystemMessage)), len(messages))
if len(system_messages) == first_non_system_index:
# A number of strict chat templates accept *one* system message, and only
# at index zero. Two adjacent leading system messages are therefore not a
# valid no-op case: LangChain's agent prompt followed by DeerFlow's
# transient compaction summary must still be folded into one message.
if len(system_messages) == 1 and first_non_system_index == 1:
return messages
merged = system_messages[0].model_copy(
@@ -46,20 +49,38 @@ def normalize_langchain_messages(messages: list[BaseMessage]) -> list[BaseMessag
return [merged, *(message for message in messages if not isinstance(message, SystemMessage))]
def normalize_model_request_messages(
system_message: SystemMessage | None,
messages: list[BaseMessage],
) -> tuple[SystemMessage | None, list[BaseMessage]]:
"""Normalize LangChain's split ``ModelRequest`` message representation.
``ModelRequest.messages`` is documented as excluding the system prompt.
Transient middleware (notably context compaction) can nevertheless place a
``SystemMessage`` in that list. Fold every such message into the dedicated
system slot so adapters without a DeerFlow-specific payload patch also
receive exactly one leading system message.
"""
if system_message is None and not any(isinstance(message, SystemMessage) for message in messages):
return None, messages
combined = [system_message, *messages] if system_message is not None else messages
normalized = normalize_langchain_messages(combined)
if normalized and isinstance(normalized[0], SystemMessage):
return normalized[0], normalized[1:]
return None, normalized
def _normalize_payload_message_list(messages: list[Any]) -> list[Any]:
system_messages = [message for message in messages if isinstance(message, dict) and message.get("role") == "system"]
if not system_messages:
return messages
first_non_system_index = next(
(
idx
for idx, message in enumerate(messages)
if not (isinstance(message, dict) and message.get("role") == "system")
),
(idx for idx, message in enumerate(messages) if not (isinstance(message, dict) and message.get("role") == "system")),
len(messages),
)
if len(system_messages) == first_non_system_index:
if len(system_messages) == 1 and first_non_system_index == 1:
return messages
merged_system = dict(system_messages[0])
@@ -19,8 +19,9 @@ re-inject thinking metadata back into the outgoing payload for any assistant
message that originally carried it:
- ``thought_signature`` for Gemini-compatible tool calls.
- ``reasoning_content`` for providers (for example DeepSeek) that require it
to be echoed in multi-turn requests when thinking mode is enabled.
- ``reasoning_content`` and the newer ``reasoning`` spelling used by vLLM and
other OpenAI-compatible providers. Both are normalized to
``additional_kwargs.reasoning_content`` for DeerFlow's existing clients.
"""
from __future__ import annotations
@@ -166,7 +167,7 @@ class PatchedChatOpenAI(ChatOpenAI):
default_chunk_class: type,
base_generation_info: dict | None,
) -> ChatGenerationChunk | None:
"""Preserve streamed ``reasoning_content`` in ``additional_kwargs``."""
"""Preserve streamed ``reasoning_content``/``reasoning`` metadata."""
# Some compatible gateways return a vendor error event followed by a
# normal-looking usage chunk, all under HTTP 200. Detect the first
# event before LangChain converts it into an empty AIMessageChunk.
@@ -174,8 +175,8 @@ class PatchedChatOpenAI(ChatOpenAI):
generation_chunk = super()._convert_chunk_to_generation_chunk(
chunk,
default_chunk_class,
base_generation_info,
)
base_generation_info,
)
if generation_chunk is None:
return None
@@ -183,11 +184,12 @@ class PatchedChatOpenAI(ChatOpenAI):
choice = choices[0] if choices else {}
delta = choice.get("delta") if isinstance(choice, dict) else None
reasoning_content = _extract_reasoning_content(delta if isinstance(delta, dict) else None)
if reasoning_content and isinstance(generation_chunk.message, AIMessageChunk):
generation_chunk.message = _append_reasoning_content(
generation_chunk.message,
reasoning_content,
)
if reasoning_content and isinstance(generation_chunk.message, AIMessageChunk):
generation_chunk.message = _append_reasoning_content(
generation_chunk.message,
reasoning_content,
reasoning=delta.get("reasoning") if isinstance(delta, dict) else None,
)
return generation_chunk
def _create_chat_result(
@@ -195,20 +197,24 @@ class PatchedChatOpenAI(ChatOpenAI):
response: dict | Any,
generation_info: dict | None = None,
) -> ChatResult:
"""Preserve non-stream ``reasoning_content`` in ``additional_kwargs``."""
"""Preserve non-stream ``reasoning_content``/``reasoning`` metadata."""
response_dict = response if isinstance(response, dict) else response.model_dump()
_raise_for_application_error(response_dict)
result = super()._create_chat_result(response, generation_info)
choices = response_dict.get("choices", [])
generations: list[ChatGeneration] = []
generations: list[ChatGeneration] = []
for index, generation in enumerate(result.generations):
message = generation.message
choice = choices[index] if index < len(choices) else {}
choice_message = choice.get("message", {}) if isinstance(choice, dict) else {}
reasoning_content = _extract_reasoning_content(choice_message if isinstance(choice_message, dict) else None)
if reasoning_content and isinstance(message, AIMessage):
message = _append_reasoning_content(message, reasoning_content)
if reasoning_content and isinstance(message, AIMessage):
message = _append_reasoning_content(
message,
reasoning_content,
reasoning=choice_message.get("reasoning") if isinstance(choice_message, dict) else None,
)
generation = ChatGeneration(
message=message,
generation_info=generation.generation_info,
@@ -259,34 +265,46 @@ def _restore_tool_call_signatures(payload_msg: dict, orig_msg: AIMessage) -> Non
payload_tc["thought_signature"] = sig
def _restore_reasoning_content(payload_msg: dict, orig_msg: AIMessage) -> None:
"""Re-inject ``reasoning_content`` onto assistant payload message.
Some thinking-enabled OpenAI-compatible providers require each historical
assistant turn to include the original ``reasoning_content`` in subsequent
requests. LangChain stores it in ``AIMessage.additional_kwargs`` but does
not always serialize it back into ``messages``.
"""
reasoning_content = orig_msg.additional_kwargs.get("reasoning_content")
if reasoning_content is not None:
payload_msg["reasoning_content"] = reasoning_content
def _extract_reasoning_content(payload: dict | None) -> str | None:
"""Extract reasoning text from provider payload delta/message objects."""
if not isinstance(payload, dict):
return None
reasoning_content = payload.get("reasoning_content")
return reasoning_content if isinstance(reasoning_content, str) and reasoning_content else None
def _append_reasoning_content(message: AIMessage | AIMessageChunk, reasoning_content: str) -> AIMessage | AIMessageChunk:
"""Append streamed reasoning chunks, dedupe complete response reasoning."""
additional_kwargs = dict(message.additional_kwargs)
existing = additional_kwargs.get("reasoning_content")
def _restore_reasoning_content(payload_msg: dict, orig_msg: AIMessage) -> None:
"""Re-inject provider reasoning onto an assistant history message.
Some thinking-enabled OpenAI-compatible providers require each historical
assistant turn to include its original reasoning in subsequent requests.
Preserve the spelling returned by the provider when available; otherwise
retain the established ``reasoning_content`` behavior.
"""
reasoning = orig_msg.additional_kwargs.get("reasoning")
if reasoning is not None:
payload_msg["reasoning"] = reasoning
return
reasoning_content = orig_msg.additional_kwargs.get("reasoning_content")
if reasoning_content is not None:
payload_msg["reasoning_content"] = reasoning_content
def _extract_reasoning_content(payload: dict | None) -> str | None:
"""Extract either supported reasoning spelling from provider payloads."""
if not isinstance(payload, dict):
return None
reasoning_content = payload.get("reasoning_content") or payload.get("reasoning")
return reasoning_content if isinstance(reasoning_content, str) and reasoning_content else None
def _append_reasoning_content(
message: AIMessage | AIMessageChunk,
reasoning_content: str,
*,
reasoning: Any = None,
) -> AIMessage | AIMessageChunk:
"""Normalize reasoning for clients while retaining its provider spelling."""
additional_kwargs = dict(message.additional_kwargs)
existing = additional_kwargs.get("reasoning_content")
if isinstance(existing, str) and existing:
if reasoning_content not in existing:
additional_kwargs["reasoning_content"] = f"{existing}{reasoning_content}"
else:
additional_kwargs["reasoning_content"] = reasoning_content
return message.model_copy(update={"additional_kwargs": additional_kwargs})
else:
additional_kwargs["reasoning_content"] = reasoning_content
if reasoning is not None:
additional_kwargs["reasoning"] = reasoning
return message.model_copy(update={"additional_kwargs": additional_kwargs})
@@ -0,0 +1,140 @@
"""Normalize OpenAI-compatible thinking fields across provider versions.
Older vLLM / DeepSeek put thinking in ``reasoning_content``. Newer vLLM
reasoning parsers put it in ``reasoning``. DeerFlow's UI and downstream code
canonicalize on ``AIMessage.additional_kwargs.reasoning_content``.
Only fields that actually appeared on a response are echoed back on later
turns, so models that still use ``reasoning_content`` (or send neither) are
unchanged.
"""
from __future__ import annotations
import json
from typing import Any
from langchain_core.messages import AIMessage, AIMessageChunk
def reasoning_to_text(reasoning: Any) -> str:
"""Best-effort extraction of readable reasoning text from provider payloads."""
if isinstance(reasoning, str):
return reasoning
if isinstance(reasoning, list):
parts = [reasoning_to_text(item) for item in reasoning]
return "".join(part for part in parts if part)
if isinstance(reasoning, dict):
for key in ("text", "content", "reasoning"):
value = reasoning.get(key)
if isinstance(value, str):
return value
if value is not None:
text = reasoning_to_text(value)
if text:
return text
try:
return json.dumps(reasoning, ensure_ascii=False)
except TypeError:
return str(reasoning)
if reasoning is None:
return ""
try:
return json.dumps(reasoning, ensure_ascii=False)
except TypeError:
return str(reasoning)
def extract_reasoning_text(payload: dict | None) -> str | None:
"""Read thinking text from ``reasoning_content`` or ``reasoning``.
Prefer ``reasoning_content`` when both are present so older DeepSeek /
vLLM payloads keep their original string. Newer vLLM that only sends
``reasoning`` (string or structured) is still captured.
"""
if not isinstance(payload, dict):
return None
for key in ("reasoning_content", "reasoning"):
value = payload.get(key)
if value is None:
continue
text = reasoning_to_text(value)
if text:
return text
return None
def _merge_streamed_value(existing: Any, incoming: Any) -> Any:
if isinstance(existing, str) and isinstance(incoming, str):
if incoming and incoming not in existing:
return f"{existing}{incoming}"
return existing or incoming
return incoming if incoming is not None else existing
def apply_reasoning_from_payload(
message: AIMessage | AIMessageChunk,
payload: dict | None,
*,
append: bool = True,
) -> AIMessage | AIMessageChunk:
"""Copy provider thinking fields onto ``additional_kwargs`` for UI + echo-back."""
if not isinstance(payload, dict):
return message
additional_kwargs = dict(message.additional_kwargs)
changed = False
if "reasoning" in payload and payload.get("reasoning") is not None:
incoming = payload.get("reasoning")
existing = additional_kwargs.get("reasoning")
merged = _merge_streamed_value(existing, incoming) if append else incoming
if merged is not None and merged != existing:
additional_kwargs["reasoning"] = merged
changed = True
text = extract_reasoning_text(payload)
if text:
existing = additional_kwargs.get("reasoning_content")
if append and isinstance(existing, str) and existing:
if text not in existing:
additional_kwargs["reasoning_content"] = f"{existing}{text}"
changed = True
elif text != existing:
additional_kwargs["reasoning_content"] = text
changed = True
if not changed:
return message
return message.model_copy(update={"additional_kwargs": additional_kwargs})
def fill_ui_reasoning_content(message: AIMessage | AIMessageChunk) -> AIMessage | AIMessageChunk:
"""If LangChain stored ``reasoning`` but not the UI field, copy the text across."""
kwargs = message.additional_kwargs
reasoning = kwargs.get("reasoning")
if reasoning is None or kwargs.get("reasoning_content"):
return message
return apply_reasoning_from_payload(message, {"reasoning": reasoning}, append=False)
def restore_reasoning_fields(payload_msg: dict, orig_msg: AIMessage) -> None:
"""Re-inject the thinking field this assistant turn originally carried.
Newer vLLM stores thinking on ``reasoning`` and we also copy a UI-facing
``reasoning_content`` string from it. Echo the wire field that actually
arrived: ``reasoning`` wins when present so we do not send a synthesized
``reasoning_content`` to a parser that no longer accepts it. Older
DeepSeek / vLLM turns only have ``reasoning_content`` and stay on that
field.
"""
reasoning = orig_msg.additional_kwargs.get("reasoning")
if reasoning is not None:
payload_msg["reasoning"] = reasoning
return
reasoning_content = orig_msg.additional_kwargs.get("reasoning_content")
if reasoning_content is not None:
payload_msg["reasoning_content"] = reasoning_content
@@ -0,0 +1,123 @@
"""Concise model connection and latency diagnostics for production logs."""
from __future__ import annotations
import logging
import threading
import time
from dataclasses import dataclass
from typing import Any
from uuid import UUID
from langchain_core.callbacks import BaseCallbackHandler
logger = logging.getLogger(__name__)
@dataclass(slots=True)
class _CallTiming:
started: float
first_chunk: float | None = None
class ModelTimingCallback(BaseCallbackHandler):
"""Log model-call start, first response chunk, completion, and failure."""
def __init__(self, *, model_name: str, base_url: str | None) -> None:
self._model_name = model_name
self._base_url = base_url or "-"
self._calls: dict[str, _CallTiming] = {}
self._lock = threading.Lock()
def on_chat_model_start(
self,
serialized: dict[str, Any],
messages: list[list[Any]],
*,
run_id: UUID,
**kwargs: Any,
) -> None:
rid = str(run_id)
now = time.monotonic()
with self._lock:
self._calls[rid] = _CallTiming(started=now)
logger.info(
"LLM_CONNECT_START model=%s base_url=%s run_id=%s message_batches=%d message_count=%d",
self._model_name,
self._base_url,
rid,
len(messages),
sum(len(batch) for batch in messages),
)
def on_llm_new_token(
self,
token: str,
*,
run_id: UUID,
chunk: Any | None = None,
**kwargs: Any,
) -> None:
"""Treat the first callback as connection success, including reasoning-only chunks."""
rid = str(run_id)
now = time.monotonic()
with self._lock:
timing = self._calls.get(rid)
if timing is None or timing.first_chunk is not None:
return
timing.first_chunk = now
first_chunk_ms = int((now - timing.started) * 1000)
logger.info(
"LLM_CONNECT_OK model=%s base_url=%s run_id=%s first_chunk_ms=%d",
self._model_name,
self._base_url,
rid,
first_chunk_ms,
)
def on_llm_end(self, response: Any, *, run_id: UUID, **kwargs: Any) -> None:
self._finish(run_id=run_id, error=None)
def on_llm_error(self, error: BaseException, *, run_id: UUID, **kwargs: Any) -> None:
self._finish(run_id=run_id, error=error)
def _finish(self, *, run_id: UUID, error: BaseException | None) -> None:
rid = str(run_id)
now = time.monotonic()
with self._lock:
timing = self._calls.pop(rid, None)
if timing is None:
logger.error(
"LLM_REQUEST_FAILED model=%s base_url=%s run_id=%s error_type=%s error=%s",
self._model_name,
self._base_url,
rid,
type(error).__name__ if error else "UnknownLifecycle",
error or "missing start event",
)
return
total_ms = int((now - timing.started) * 1000)
first_chunk_ms = int((timing.first_chunk - timing.started) * 1000) if timing.first_chunk else None
if error is None:
logger.info(
"LLM_REQUEST_DONE model=%s base_url=%s run_id=%s status=success first_chunk_ms=%s total_ms=%d",
self._model_name,
self._base_url,
rid,
first_chunk_ms if first_chunk_ms is not None else "none",
total_ms,
)
return
logger.error(
"LLM_REQUEST_FAILED model=%s base_url=%s run_id=%s status=error first_chunk_ms=%s total_ms=%d error_type=%s error=%s",
self._model_name,
self._base_url,
rid,
first_chunk_ms if first_chunk_ms is not None else "none",
total_ms,
type(error).__name__,
error,
)
@@ -6,15 +6,16 @@ from assistant messages and streaming deltas. That breaks interleaved
thinking/tool-call flows because vLLM expects the assistant's prior reasoning to
be echoed back on subsequent turns.
This provider preserves ``reasoning`` on:
- non-streaming responses
- streaming deltas
- multi-turn request payloads
This provider preserves ``reasoning`` (newer vLLM) and
``reasoning_content`` (older vLLM / DeepSeek-style parsers) on:
- non-streaming responses
- streaming deltas
- multi-turn request payloads
UI code still reads the canonical ``additional_kwargs.reasoning_content``.
"""
from __future__ import annotations
import json
from collections.abc import Mapping
from typing import Any, cast
@@ -32,11 +33,12 @@ from langchain_core.messages import (
)
from langchain_core.messages.tool import tool_call_chunk
from langchain_core.outputs import ChatGeneration, ChatGenerationChunk, ChatResult
from langchain_openai import ChatOpenAI
from langchain_openai.chat_models.base import _create_usage_metadata
from deerflow.models.message_ordering import normalize_chat_payload_messages
from langchain_openai import ChatOpenAI
from langchain_openai.chat_models.base import _create_usage_metadata
from deerflow.models.message_ordering import normalize_chat_payload_messages
from deerflow.models.reasoning_fields import apply_reasoning_from_payload, restore_reasoning_fields
def _normalize_vllm_chat_template_kwargs(payload: dict[str, Any]) -> None:
"""Map DeerFlow's legacy ``thinking`` toggle to vLLM/Qwen's ``enable_thinking``.
@@ -66,31 +68,9 @@ def _normalize_vllm_chat_template_kwargs(payload: dict[str, Any]) -> None:
def _reasoning_to_text(reasoning: Any) -> str:
"""Best-effort extraction of readable reasoning text from vLLM payloads."""
if isinstance(reasoning, str):
return reasoning
from deerflow.models.reasoning_fields import reasoning_to_text
if isinstance(reasoning, list):
parts = [_reasoning_to_text(item) for item in reasoning]
return "".join(part for part in parts if part)
if isinstance(reasoning, dict):
for key in ("text", "content", "reasoning"):
value = reasoning.get(key)
if isinstance(value, str):
return value
if value is not None:
text = _reasoning_to_text(value)
if text:
return text
try:
return json.dumps(reasoning, ensure_ascii=False)
except TypeError:
return str(reasoning)
try:
return json.dumps(reasoning, ensure_ascii=False)
except TypeError:
return str(reasoning)
return reasoning_to_text(reasoning)
def _convert_delta_to_message_chunk_with_reasoning(_dict: Mapping[str, Any], default_class: type[BaseMessageChunk]) -> BaseMessageChunk:
@@ -106,12 +86,8 @@ def _convert_delta_to_message_chunk_with_reasoning(_dict: Mapping[str, Any], def
function_call["name"] = ""
additional_kwargs["function_call"] = function_call
reasoning = _dict.get("reasoning")
if reasoning is not None:
additional_kwargs["reasoning"] = reasoning
reasoning_text = _reasoning_to_text(reasoning)
if reasoning_text:
additional_kwargs["reasoning_content"] = reasoning_text
seed = AIMessageChunk(content="", additional_kwargs=additional_kwargs)
additional_kwargs = dict(apply_reasoning_from_payload(seed, dict(_dict)).additional_kwargs)
tool_call_chunks = []
if raw_tool_calls := _dict.get("tool_calls"):
@@ -151,11 +127,7 @@ def _convert_delta_to_message_chunk_with_reasoning(_dict: Mapping[str, Any], def
def _restore_reasoning_field(payload_msg: dict[str, Any], orig_msg: AIMessage) -> None:
"""Re-inject vLLM reasoning onto outgoing assistant messages."""
reasoning = orig_msg.additional_kwargs.get("reasoning")
if reasoning is None:
reasoning = orig_msg.additional_kwargs.get("reasoning_content")
if reasoning is not None:
payload_msg["reasoning"] = reasoning
restore_reasoning_fields(payload_msg, orig_msg)
class VllmChatModel(ChatOpenAI):
@@ -184,14 +156,14 @@ class VllmChatModel(ChatOpenAI):
for payload_msg, orig_msg in zip(payload_messages, original_messages):
if payload_msg.get("role") == "assistant" and isinstance(orig_msg, AIMessage):
_restore_reasoning_field(payload_msg, orig_msg)
else:
ai_messages = [message for message in original_messages if isinstance(message, AIMessage)]
assistant_payloads = [message for message in payload_messages if message.get("role") == "assistant"]
for payload_msg, ai_msg in zip(assistant_payloads, ai_messages):
_restore_reasoning_field(payload_msg, ai_msg)
normalize_chat_payload_messages(payload)
return payload
else:
ai_messages = [message for message in original_messages if isinstance(message, AIMessage)]
assistant_payloads = [message for message in payload_messages if message.get("role") == "assistant"]
for payload_msg, ai_msg in zip(assistant_payloads, ai_messages):
_restore_reasoning_field(payload_msg, ai_msg)
normalize_chat_payload_messages(payload)
return payload
def _create_chat_result(self, response: dict | openai.BaseModel, generation_info: dict | None = None) -> ChatResult:
"""Preserve vLLM reasoning on non-streaming responses."""
@@ -204,13 +176,12 @@ class VllmChatModel(ChatOpenAI):
message = generation.message
if not isinstance(message, AIMessage):
continue
reasoning = choice.get("message", {}).get("reasoning")
if reasoning is None:
continue
message.additional_kwargs["reasoning"] = reasoning
reasoning_text = _reasoning_to_text(reasoning)
if reasoning_text:
message.additional_kwargs["reasoning_content"] = reasoning_text
choice_message = choice.get("message", {}) if isinstance(choice, dict) else {}
generation.message = apply_reasoning_from_payload(
message,
choice_message if isinstance(choice_message, dict) else None,
append=False,
)
return result