sync: 鍚屾灞€鍩熺綉鏈€鏂颁唬鐮?(source master @ b69568e4)

This commit is contained in:
gozeros committed 2026-10-21 14:51:31 +08:00
1 parent 4f7f07504a
commit 8bc8b0e082
1651 files changed
+278599 -11416

No files matched your search

@@ -0,0 +1,106 @@
"""Invocation-local text and completion checks for compatible chat streams."""
from __future__ import annotations
from collections.abc import Mapping
from dataclasses import dataclass, field
from typing import Any
def needs_qwen_flash_next_integrity(*model_names: str | None) -> bool:
"""Match the affected model id or its local alias, without broad Qwen rules."""
return any(
name.strip().lower().replace("_", "-").rsplit("/", 1)[-1] == "qwen3.8-flash-next"
for name in model_names
if isinstance(name, str)
)
class IncompleteChatStreamError(RuntimeError):
"""A provider stream ended before a complete response could be committed.
Only a fixed reason is exposed. Provider text, URLs and error bodies must
never appear in an exception that the gateway may send to a browser.
"""
def __init__(self, reason: str) -> None:
self.reason = reason
super().__init__(f"Incomplete OpenAI-compatible chat stream: {reason}")
@dataclass
class ChatStreamState:
"""Track one request, never a model instance or a persisted conversation."""
observed_chat_frame: bool = False
terminal_received: bool = False
content: dict[int, str] = field(default_factory=dict)
reasoning: dict[int, str] = field(default_factory=dict)
def observe(self, choices: list[dict[str, Any]]) -> None:
self.observed_chat_frame = True
# LangChain consumes the first choice, so another choice's finish
# reason must not make an unfinished first choice look complete.
if choices and choices[0].get("finish_reason"):
self.terminal_received = True
def ensure_finished(self) -> None:
# Responses API and adapter-generated chunks do not pass through the
# chat-completions converter and have their own completion semantics.
if self.observed_chat_frame and not self.terminal_received:
raise IncompleteChatStreamError("missing_terminal_frame")
def normalize_text(
self,
*,
index: int,
delta: dict[str, Any],
original_delta: Mapping[str, Any] | None,
message: Mapping[str, Any] | None,
) -> None:
"""Convert explicit full-message snapshots to suffixes.
A real delta always wins and is appended verbatim, including repeated
words. Only text supplied through ``message`` is treated as a full
snapshot; prefix guessing on ordinary deltas would lose real tokens.
"""
for fields, accumulated in (
(("content",), self.content),
(("reasoning_content", "reasoning"), self.reasoning),
):
source = original_delta or {}
current = accumulated.get(index, "")
incoming = _first_text(source, fields)
if incoming is not None:
accumulated[index] = current + incoming[1]
continue
# Structured content and vendor reasoning objects are opaque.
# Preserve them instead of replacing them with a text snapshot.
if any(source.get(name) is not None and not isinstance(source[name], str) for name in fields):
continue
snapshot = _first_text(message or {}, fields)
if snapshot is None:
continue
name, text = snapshot
if text.startswith(current):
suffix = text[len(current):]
accumulated[index] = text
elif current.startswith(text):
# A duplicate/stale snapshot adds nothing to the live text.
suffix = ""
else:
# A delta cannot retract already-yielded bytes. Retry before
# committing contradictory text into the conversation.
raise IncompleteChatStreamError("inconsistent_message_snapshot")
for alias in fields:
delta.pop(alias, None)
delta[name] = suffix
def _first_text(source: Mapping[str, Any], fields: tuple[str, ...]) -> tuple[str, str] | None:
for name in fields:
value = source.get(name)
if isinstance(value, str) and value:
return name, value
return None
@@ -0,0 +1,50 @@
"""Shared detection for adaptive context recovery (no assumed model window)."""
from collections.abc import Mapping
CONTEXT_RECOVERY_MESSAGE = "请补充本次需要处理的具体问题或关键材料,我会继续为你解答。"
_PATTERNS = (
"context_length_exceeded",
"maximum context length",
"max context length",
"context window exceeds limit",
"context window exceeded",
"exceeds the context window",
"max_model_len",
"prompt is too long",
"input is too long",
"too many tokens",
"input token count",
"prompt token count",
"exceeds the model's context",
"exceeds model context",
"上下文长度",
"最大上下文",
"输入 token 数超过",
"输入token数超过",
)
def is_context_length_error(exc: BaseException) -> bool:
"""Inspect provider error codes as well as text, without matching all 400s."""
values = [str(exc), str(getattr(exc, "code", "")), str(getattr(exc, "error_code", ""))]
# Compatible gateways may use numeric codes and put the only context-size
# signal in their message. Our adapter keeps that text private so __str__
# and framework logs cannot expose provider bodies; classify it in-process.
classification_detail = getattr(exc, "_classification_detail", None)
if isinstance(classification_detail, str):
values.append(classification_detail)
body = getattr(exc, "body", None)
if isinstance(body, Mapping):
error = body.get("error", body)
if isinstance(error, Mapping):
values.extend(str(error.get(key, "")) for key in ("code", "type", "message"))
return any(pattern in value.lower() for value in values for pattern in _PATTERNS)
class ContextRecoveryExhausted(RuntimeError):
"""Internal failure for contracts that cannot degrade to conversational text."""
def __init__(self) -> None:
super().__init__("context_length_exceeded: adaptive recovery exhausted")
@@ -2,8 +2,11 @@ import logging
from langchain.chat_models import BaseChatModel
from deerflow.config import get_app_config
from deerflow.config.app_config import AppConfig
from deerflow.config import get_app_config
from deerflow.config.app_config import AppConfig
from deerflow.config.model_config import WriteFileTransport
from deerflow.config.system_settings import get_write_file_compatibility_settings
from deerflow.models.chat_stream_state import needs_qwen_flash_next_integrity
from deerflow.reflection import resolve_class
from deerflow.tracing import build_tracing_callbacks
@@ -31,6 +34,92 @@ def _vllm_disable_chat_template_kwargs(chat_template_kwargs: dict) -> dict:
return disable_kwargs
# Qwen3.8 chat templates only accept these three effort names.
_QWEN_REASONING_EFFORT = {
"minimal": "low",
"low": "low",
"medium": "medium",
"high": "xhigh",
"xhigh": "xhigh",
}
_OPENAI_THINKING_OFF_EXTRA_BODY = {
"enable_thinking": False,
"chat_template_kwargs": {"enable_thinking": False},
"thinking": {"type": "disabled"},
}
def _is_vllm_provider(model_use: str) -> bool:
lowered = (model_use or "").lower()
return "vllm_provider" in lowered or lowered.endswith(":vllmchatmodel")
def _is_qwen38_model(model_config) -> bool:
blob = f"{getattr(model_config, 'name', '')} {getattr(model_config, 'model', '')}".lower()
return "qwen3.8" in blob
def _should_control_openai_thinking(model_config) -> bool:
"""Whether this OpenAI-compatible model needs explicit thinking on/off flags.
Qwen3.8 (and current vLLM reasoning parsers) default to thinking at ``xhigh``.
If DeerFlow does not send ``enable_thinking: false``, the first visible answer
token waits until that hidden reasoning finishes — third-party chat UIs that
disable thinking or stream the reasoning field therefore feel much faster.
"""
if getattr(model_config, "supports_thinking", False):
return True
if _is_vllm_provider(getattr(model_config, "use", "")):
return True
return _is_qwen38_model(model_config)
def _map_qwen_reasoning_effort(effort: object) -> str | None:
if effort is None:
return None
return _QWEN_REASONING_EFFORT.get(str(effort).strip().lower())
def _apply_openai_thinking_control(
model_config,
model_class,
model_settings_from_config: dict,
*,
thinking_enabled: bool,
) -> None:
"""Inject vLLM/Qwen thinking flags when the UI toggle is on or off."""
if "extra_body" not in getattr(model_class, "model_fields", {}):
return
qwen_vllm = _is_vllm_provider(getattr(model_config, "use", "")) or _is_qwen38_model(model_config)
if thinking_enabled:
if not qwen_vllm:
return
enable_body: dict = {
"enable_thinking": True,
"chat_template_kwargs": {"enable_thinking": True},
}
model_settings_from_config["extra_body"] = _deep_merge_dicts(
model_settings_from_config.get("extra_body"),
enable_body,
)
return
if not _should_control_openai_thinking(model_config):
return
disable_body = (
{
"enable_thinking": False,
"chat_template_kwargs": {"enable_thinking": False},
}
if qwen_vllm
else _OPENAI_THINKING_OFF_EXTRA_BODY
)
model_settings_from_config["extra_body"] = _deep_merge_dicts(
model_settings_from_config.get("extra_body"),
disable_body,
)
def _enable_stream_usage_by_default(model_use_path: str, model_settings_from_config: dict) -> None:
"""Enable stream usage for OpenAI-compatible models unless explicitly configured.
@@ -92,9 +181,50 @@ def create_chat_model(name: str | None = None, thinking_enabled: bool = False, *
"when_thinking_enabled",
"when_thinking_disabled",
"thinking",
"supports_vision",
},
)
"supports_vision",
"write_file_transport",
},
)
configured_write_file_transport = model_config.write_file_transport
runtime_write_file_compatibility = get_write_file_compatibility_settings()
effective_write_file_transport = (
WriteFileTransport.BUFFERED_GATEWAY
if runtime_write_file_compatibility.enabled
else configured_write_file_transport
)
if effective_write_file_transport in {
WriteFileTransport.BUFFERED_GATEWAY,
WriteFileTransport.ARTIFACT_RENDERER,
WriteFileTransport.TEXT_BRIDGE,
}:
from deerflow.models.patched_openai import PatchedChatOpenAI
if not issubclass(model_class, PatchedChatOpenAI):
raise ValueError(
"write_file_transport=buffered_gateway/artifact_renderer/text_bridge is supported only by "
"langchain_openai:ChatOpenAI or PatchedChatOpenAI models"
)
if model_config.use_responses_api:
raise ValueError(
"write_file_transport=buffered_gateway/artifact_renderer/text_bridge requires the "
"OpenAI-compatible chat-completions API, not use_responses_api"
)
# The admin runtime switch overrides every compatible model on its next
# construction. With the switch off, retain the legacy per-model value.
model_settings_from_config["write_file_transport"] = effective_write_file_transport.value
# ``max_input_tokens`` is a DeerFlow-side safety limit, not an OpenAI API
# request parameter. ModelConfig intentionally allows provider-specific
# extras, so without removing it here ChatOpenAI would forward the unknown
# key to some compatible gateways and turn the protection itself into a
# HTTP 400. Mirror it into LangChain's local-only ``profile`` metadata
# when the selected model supports that field; the final context guard
# reads the original model config and therefore also works for adapters
# without profiles.
configured_max_input_tokens = model_settings_from_config.pop("max_input_tokens", None)
if configured_max_input_tokens is not None and "profile" in getattr(model_class, "model_fields", {}):
profile = dict(model_settings_from_config.get("profile") or {})
profile["max_input_tokens"] = configured_max_input_tokens
model_settings_from_config["profile"] = profile
# Compute effective when_thinking_enabled by merging in the `thinking` shortcut field.
# The `thinking` shortcut is equivalent to setting when_thinking_enabled["thinking"].
has_thinking_settings = (model_config.when_thinking_enabled is not None) or (model_config.thinking is not None)
@@ -107,6 +237,13 @@ def create_chat_model(name: str | None = None, thinking_enabled: bool = False, *
raise ValueError(f"Model {name} does not support thinking. Set `supports_thinking` to true in the `config.yaml` to enable thinking.") from None
if effective_wte:
model_settings_from_config.update(effective_wte)
elif thinking_enabled and model_config.supports_thinking and model_config.use == "langchain_openai:ChatOpenAI":
# vLLM/Qwen OpenAI-compatible endpoints commonly require this explicit
# request flag before they emit delta.reasoning.
model_settings_from_config["extra_body"] = _deep_merge_dicts(
model_settings_from_config.get("extra_body"),
{"enable_thinking": True},
)
if not thinking_enabled:
if model_config.when_thinking_disabled is not None:
# User-provided disable settings take full precedence
@@ -127,9 +264,25 @@ def create_chat_model(name: str | None = None, thinking_enabled: bool = False, *
elif has_thinking_settings and effective_wte.get("thinking", {}).get("type"):
# Native langchain_anthropic: thinking is a direct constructor parameter
model_settings_from_config["thinking"] = {"type": "disabled"}
if not model_config.supports_reasoning_effort:
kwargs.pop("reasoning_effort", None)
# The Gateway validates the database-backed capability before constructing
# the graph. Honor that server-validated runtime value without consulting a
# legacy config.yaml flag. Qwen/vLLM accepts low/medium/xhigh; normalize the
# old UI aliases defensively for internal callers that do not traverse the
# public run boundary.
requested_reasoning_effort = kwargs.get("reasoning_effort")
if requested_reasoning_effort is not None:
model_settings_from_config.pop("reasoning_effort", None)
if _is_vllm_provider(model_config.use) or _is_qwen38_model(model_config):
mapped_effort = _map_qwen_reasoning_effort(requested_reasoning_effort)
if mapped_effort is not None:
kwargs["reasoning_effort"] = mapped_effort
_apply_openai_thinking_control(
model_config,
model_class,
model_settings_from_config,
thinking_enabled=thinking_enabled,
)
# Hard override: force-disable thinking regardless of the per-model config shape.
# Injected only for OpenAI-compatible classes (those declaring `extra_body`);
@@ -139,15 +292,10 @@ def create_chat_model(name: str | None = None, thinking_enabled: bool = False, *
# (DeepSeek-V3.1+/Qwen3/GLM hybrid models on api.siliconflow.cn — the knob the
# nested ``chat_template_kwargs`` shape does NOT reach), vLLM/Qwen native
# ``chat_template_kwargs.enable_thinking``, and Zhipu GLM ``thinking.type``.
# Only models that explicitly opt into thinking should receive vendor-specific
# disable payloads. Generic OpenAI-compatible gateways (including LiteLLM)
# may reject these otherwise-unknown fields instead of forwarding them.
if (
force_disable_thinking
and not thinking_enabled
and model_config.supports_thinking
and "extra_body" in getattr(model_class, "model_fields", {})
):
# Only models that explicitly opt into thinking should receive vendor-specific
# disable payloads. Generic OpenAI-compatible gateways (including LiteLLM)
# may reject these otherwise-unknown fields instead of forwarding them.
if force_disable_thinking and not thinking_enabled and model_config.supports_thinking and "extra_body" in getattr(model_class, "model_fields", {}):
model_settings_from_config["extra_body"] = _deep_merge_dicts(
model_settings_from_config.get("extra_body"),
{"enable_thinking": False, "chat_template_kwargs": {"enable_thinking": False}, "thinking": {"type": "disabled"}},
@@ -186,11 +334,35 @@ def create_chat_model(name: str | None = None, thinking_enabled: bool = False, *
if "stream_usage" in getattr(model_class, "model_fields", {}):
model_settings_from_config["stream_usage"] = True
model_instance = model_class(**kwargs, **model_settings_from_config)
# The deployment may expose the affected model under a generic provider id.
# Enable its local stream checks by either id or configured name, while
# honoring an explicit override and keeping other models on their old path.
if "strict_stream_integrity" in getattr(model_class, "model_fields", {}):
if "strict_stream_integrity" in kwargs:
model_settings_from_config["strict_stream_integrity"] = kwargs.pop("strict_stream_integrity")
elif needs_qwen_flash_next_integrity(model_config.name, model_config.model):
model_settings_from_config.setdefault("strict_stream_integrity", True)
callbacks = build_tracing_callbacks()
if callbacks:
existing_callbacks = model_instance.callbacks or []
model_instance = model_class(**kwargs, **model_settings_from_config)
# Always emit a compact model lifecycle trace. In particular, the first
# streamed callback proves that the upstream endpoint accepted the request
# and started returning data, including reasoning-only chunks whose normal
# content string is empty.
from deerflow.models.timing_callback import ModelTimingCallback
existing_callbacks = model_instance.callbacks or []
model_instance.callbacks = [
*existing_callbacks,
ModelTimingCallback(
model_name=name,
base_url=str(model_config.base_url) if model_config.base_url else None,
),
]
callbacks = build_tracing_callbacks()
if callbacks:
existing_callbacks = model_instance.callbacks or []
model_instance.callbacks = [*existing_callbacks, *callbacks]
logger.debug(f"Tracing attached to model '{name}' with providers={len(callbacks)}")
return model_instance
@@ -6,7 +6,6 @@ from typing import Any
from langchain_core.messages import BaseMessage, SystemMessage
_SYSTEM_SEPARATOR = "\n\n---\n\n"
@@ -35,7 +34,11 @@ def normalize_langchain_messages(messages: list[BaseMessage]) -> list[BaseMessag
return messages
first_non_system_index = next((idx for idx, message in enumerate(messages) if not isinstance(message, SystemMessage)), len(messages))
if len(system_messages) == first_non_system_index:
# A number of strict chat templates accept *one* system message, and only
# at index zero. Two adjacent leading system messages are therefore not a
# valid no-op case: LangChain's agent prompt followed by DeerFlow's
# transient compaction summary must still be folded into one message.
if len(system_messages) == 1 and first_non_system_index == 1:
return messages
merged = system_messages[0].model_copy(
@@ -46,20 +49,38 @@ def normalize_langchain_messages(messages: list[BaseMessage]) -> list[BaseMessag
return [merged, *(message for message in messages if not isinstance(message, SystemMessage))]
def normalize_model_request_messages(
system_message: SystemMessage | None,
messages: list[BaseMessage],
) -> tuple[SystemMessage | None, list[BaseMessage]]:
"""Normalize LangChain's split ``ModelRequest`` message representation.
``ModelRequest.messages`` is documented as excluding the system prompt.
Transient middleware (notably context compaction) can nevertheless place a
``SystemMessage`` in that list. Fold every such message into the dedicated
system slot so adapters without a DeerFlow-specific payload patch also
receive exactly one leading system message.
"""
if system_message is None and not any(isinstance(message, SystemMessage) for message in messages):
return None, messages
combined = [system_message, *messages] if system_message is not None else messages
normalized = normalize_langchain_messages(combined)
if normalized and isinstance(normalized[0], SystemMessage):
return normalized[0], normalized[1:]
return None, normalized
def _normalize_payload_message_list(messages: list[Any]) -> list[Any]:
system_messages = [message for message in messages if isinstance(message, dict) and message.get("role") == "system"]
if not system_messages:
return messages
first_non_system_index = next(
(
idx
for idx, message in enumerate(messages)
if not (isinstance(message, dict) and message.get("role") == "system")
),
(idx for idx, message in enumerate(messages) if not (isinstance(message, dict) and message.get("role") == "system")),
len(messages),
)
if len(system_messages) == first_non_system_index:
if len(system_messages) == 1 and first_non_system_index == 1:
return messages
merged_system = dict(system_messages[0])
@@ -0,0 +1,490 @@
"""Keep one day of the exact text sent to a model and the text it sent back.
The admin page reads this store. Rows older than a day are dropped. Bodies are
capped so a long tool argument cannot grow the file without limit. Recording
never raises into the model call.
"""
from __future__ import annotations
import contextvars
import json
import logging
import os
import tempfile
import threading
import uuid
from collections.abc import Mapping
from dataclasses import dataclass, field
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
logger = logging.getLogger(__name__)
_RETENTION = timedelta(days=1)
_STORE_NAME = "model-exchanges.jsonl"
_MAX_RECORDS = 100
_MAX_FIELD_CHARS = 200_000
_PREVIEW_CHARS = 80
_TRUNCATED = "\n…(超出保存长度,后面已截断)"
_LOCK = threading.Lock()
_ACTIVE: contextvars.ContextVar[ExchangeCapture | None] = contextvars.ContextVar(
"deerflow_model_exchange",
default=None,
)
_store_override: Path | None = None
@dataclass
class ExchangeCapture:
"""One in-flight model call. Text stays here until the call finishes."""
model: str = ""
enable_thinking: str = "unset"
request_text: str = ""
request_body: str = ""
request_preview: str = ""
content_parts: list[str] = field(default_factory=list)
reasoning_by_field: dict[str, list[str]] = field(default_factory=dict)
reasoning_fields: list[str] = field(default_factory=list)
raw_chunks: list[str] = field(default_factory=list)
owned: bool = False
token: contextvars.Token | None = None
saved: bool = False
def use_exchange_store(path: Path | None) -> None:
"""Point the store at a test file. ``None`` restores the runtime data dir."""
global _store_override
_store_override = path
def open_exchange() -> ExchangeCapture:
return ExchangeCapture()
def bind_exchange(capture: ExchangeCapture) -> contextvars.Token:
return _ACTIVE.set(capture)
def unbind_exchange(token: contextvars.Token) -> None:
_ACTIVE.reset(token)
def activate_exchange() -> tuple[ExchangeCapture, contextvars.Token]:
capture = open_exchange()
return capture, bind_exchange(capture)
def deactivate_exchange(capture: ExchangeCapture, token: contextvars.Token) -> None:
try:
finish_exchange(capture)
finally:
unbind_exchange(token)
def remember_request(payload: Mapping[str, Any]) -> None:
"""Copy the outbound messages once, after the payload is final."""
try:
capture = _ACTIVE.get()
if capture is None:
capture = ExchangeCapture(owned=True)
capture.token = _ACTIVE.set(capture)
elif capture.owned and capture.request_text and not capture.saved:
_discard_owned(capture)
capture = ExchangeCapture(owned=True)
capture.token = _ACTIVE.set(capture)
if capture.request_text:
return
messages = payload.get("messages")
capture.model = str(payload.get("model") or "")
capture.enable_thinking = _thinking_flag(payload)
capture.request_preview = _preview(_last_user_text(messages))
capture.request_text = _request_text(payload, messages)
capture.request_body = _replay_request(payload)
except Exception:
logger.debug("model exchange request was not captured", exc_info=True)
def remember_provider_chunk(chunk: Mapping[str, Any]) -> None:
"""Append one provider chunk exactly as it arrived."""
try:
capture = _ACTIVE.get()
if capture is None:
return
_store_raw(capture, chunk)
for choice in _choices(chunk):
delta = choice.get("delta") if isinstance(choice.get("delta"), Mapping) else {}
message = choice.get("message") if isinstance(choice.get("message"), Mapping) else {}
_append_channel(capture.content_parts, _text_field(delta, "content") or _text_field(message, "content"))
_take_reasoning(capture, delta)
_take_reasoning(capture, message)
except Exception:
logger.debug("model exchange chunk was not captured", exc_info=True)
def remember_raw_response(payload: Mapping[str, Any]) -> None:
"""Keep a non-streaming response body exactly as the provider sent it."""
try:
capture = _ACTIVE.get()
if capture is None:
return
_store_raw(capture, payload, only_if_empty=True)
except Exception:
logger.debug("model exchange response was not captured", exc_info=True)
def remember_final_message(message: Mapping[str, Any]) -> None:
"""Record a non-streaming message when no chunks were captured."""
try:
capture = _ACTIVE.get()
if capture is None:
return
_store_raw(capture, message, only_if_empty=True)
if capture.content_parts or _has_reasoning(capture):
return
_append_channel(capture.content_parts, _text_field(message, "content"))
_take_reasoning(capture, message)
except Exception:
logger.debug("model exchange message was not captured", exc_info=True)
def finish_owned_exchange() -> None:
capture = _ACTIVE.get()
if capture is None or not capture.owned:
return
token = capture.token
try:
finish_exchange(capture)
finally:
if token is not None:
_ACTIVE.reset(token)
def finish_exchange(capture: ExchangeCapture) -> None:
if capture.saved or not capture.request_text:
capture.saved = True
return
capture.saved = True
content = "".join(capture.content_parts)
reasoning_by_field = {
name: "".join(capture.reasoning_by_field.get(name, []))
for name in capture.reasoning_fields
if "".join(capture.reasoning_by_field.get(name, []))
}
reasoning = "".join(reasoning_by_field.values())
reasoning_field = "、".join(reasoning_by_field) or "(没有单独的思考字段)"
response_raw = _cap("\n\n".join(capture.raw_chunks))
preview_source = content or reasoning
record = {
"id": uuid.uuid4().hex,
"at": datetime.now(UTC).isoformat(timespec="seconds"),
"model": capture.model,
"user_id": _current_user_id(),
"enable_thinking": capture.enable_thinking,
"request_preview": capture.request_preview,
"response_preview": _preview(preview_source),
"request_text": _cap(capture.request_text),
"request_body": capture.request_body,
"response_text": response_raw,
"response_raw": response_raw,
"response_content": _cap(content),
"response_reasoning": _cap(reasoning),
"reasoning_field": reasoning_field,
}
_append_record(record)
def query_model_exchanges() -> list[dict[str, Any]]:
"""Newest first. Entries older than one day are dropped."""
with _LOCK:
records = _read_records()
_write_records(records)
return list(reversed(records))
def _discard_owned(capture: ExchangeCapture) -> None:
capture.saved = True
token = capture.token
if token is not None:
_ACTIVE.reset(token)
def _store_path() -> Path:
if _store_override is not None:
return _store_override
from deerflow.config.paths import get_paths
return get_paths().base_dir / _STORE_NAME
def _append_record(record: dict[str, Any]) -> None:
try:
with _LOCK:
records = _read_records()
records.append(record)
if len(records) > _MAX_RECORDS:
records = records[-_MAX_RECORDS:]
_write_records(records)
except Exception:
logger.debug("model exchange was not saved", exc_info=True)
def _read_records() -> list[dict[str, Any]]:
path = _store_path()
if not path.is_file():
return []
cutoff = datetime.now(UTC) - _RETENTION
kept: list[dict[str, Any]] = []
for line in path.read_text(encoding="utf-8").splitlines():
if not line.strip():
continue
try:
record = json.loads(line)
except json.JSONDecodeError:
continue
if isinstance(record, dict) and _record_time(record) >= cutoff:
kept.append(record)
if len(kept) > _MAX_RECORDS:
kept = kept[-_MAX_RECORDS:]
return kept
def _write_records(records: list[dict[str, Any]]) -> None:
path = _store_path()
path.parent.mkdir(parents=True, exist_ok=True)
fd, tmp = tempfile.mkstemp(dir=path.parent, prefix=".model-exchanges.", suffix=".tmp")
try:
with os.fdopen(fd, "w", encoding="utf-8") as handle:
for record in records:
handle.write(json.dumps(record, ensure_ascii=False) + "\n")
os.replace(tmp, path)
except Exception:
try:
os.unlink(tmp)
except OSError:
pass
raise
def _record_time(record: Mapping[str, Any]) -> datetime:
raw = str(record.get("at") or "")
try:
parsed = datetime.fromisoformat(raw)
except ValueError:
return datetime.min.replace(tzinfo=UTC)
if parsed.tzinfo is None:
return parsed.replace(tzinfo=UTC)
return parsed.astimezone(UTC)
def _current_user_id() -> str:
try:
from deerflow.runtime.user_context import get_effective_user_id
return get_effective_user_id()
except Exception:
return ""
def _thinking_flag(payload: Mapping[str, Any]) -> str:
extra_body = payload.get("extra_body")
value = extra_body.get("enable_thinking", payload.get("enable_thinking")) if isinstance(extra_body, Mapping) else payload.get("enable_thinking")
if value is True:
return "true"
if value is False:
return "false"
if value is None:
return "unset"
return "other"
def _choices(chunk: Mapping[str, Any]) -> list[Mapping[str, Any]]:
choices = chunk.get("choices")
if not isinstance(choices, list) or not choices:
nested = chunk.get("chunk")
if isinstance(nested, Mapping):
choices = nested.get("choices")
if not isinstance(choices, list):
return []
return [choice for choice in choices if isinstance(choice, Mapping)]
def _has_reasoning(capture: ExchangeCapture) -> bool:
return any(parts for parts in capture.reasoning_by_field.values())
def _take_reasoning(capture: ExchangeCapture, source: Mapping[str, Any]) -> None:
"""Keep thinking text under the wire field that actually carried it."""
for name in ("reasoning_content", "reasoning"):
text = _text_field(source, name)
if not text:
continue
if name not in capture.reasoning_fields:
capture.reasoning_fields.append(name)
_append_channel(capture.reasoning_by_field.setdefault(name, []), text)
def _text_field(payload: Mapping[str, Any], key: str) -> str:
value = payload.get(key)
if isinstance(value, str):
return value
return _content_to_text(value) if key == "content" else ""
def _content_to_text(content: Any) -> str:
if isinstance(content, str):
return content
if isinstance(content, list):
parts: list[str] = []
for item in content:
if isinstance(item, str):
parts.append(item)
elif isinstance(item, Mapping):
text = item.get("text")
if isinstance(text, str) and text:
parts.append(text)
elif item.get("type") == "image_url":
parts.append("[image]")
return "\n".join(parts)
if content is None:
return ""
return str(content)
def _last_user_text(messages: Any) -> str:
if not isinstance(messages, list):
return ""
for message in reversed(messages):
if isinstance(message, Mapping) and message.get("role") == "user":
return _content_to_text(message.get("content"))
return ""
_SECRET_KEYS = frozenset({"api_key", "openai_api_key", "authorization", "token"})
def _replay_request(payload: Mapping[str, Any]) -> str:
"""JSON body the OpenAI client actually posts. ``extra_body`` is flattened."""
ready = _json_ready(payload)
if not isinstance(ready, dict):
return "{}"
extra = ready.pop("extra_body", None)
if isinstance(extra, dict):
ready.update(extra)
return _fit_replay(ready)
def _store_raw(capture: ExchangeCapture, payload: Mapping[str, Any], *, only_if_empty: bool = False) -> None:
if only_if_empty and capture.raw_chunks:
return
if sum(len(part) for part in capture.raw_chunks) >= _MAX_FIELD_CHARS:
return
capture.raw_chunks.append(json.dumps(payload, ensure_ascii=False, indent=2, default=str))
def _json_ready(value: Any) -> Any:
if value is None or isinstance(value, (str, int, float, bool)):
return value
if isinstance(value, Mapping):
return {
str(key): _json_ready(item)
for key, item in value.items()
if item is not None and str(key) not in _SECRET_KEYS
}
if isinstance(value, (list, tuple)):
return [_json_ready(item) for item in value]
return str(value)
def _fit_replay(body: dict[str, Any]) -> str:
encoded = json.dumps(body, ensure_ascii=False, indent=2)
if len(encoded) <= _MAX_FIELD_CHARS:
return encoded
tools = body.get("tools")
if isinstance(tools, list) and tools:
names: list[str] = []
for tool in tools:
function = tool.get("function") if isinstance(tool, dict) else None
name = function.get("name") if isinstance(function, dict) else None
if isinstance(name, str) and name and name not in names:
names.append(name)
body["tools"] = [{"_omitted": "工具定义过长,这里只保留工具名", "names": names}]
encoded = json.dumps(body, ensure_ascii=False, indent=2)
if len(encoded) <= _MAX_FIELD_CHARS:
return encoded
messages = body.get("messages")
if isinstance(messages, list):
for _ in range(12):
if len(encoded) <= _MAX_FIELD_CHARS:
break
longest: dict[str, Any] | None = None
longest_len = 0
for message in messages:
if not isinstance(message, dict):
continue
content = message.get("content")
if isinstance(content, str) and len(content) > longest_len:
longest = message
longest_len = len(content)
if longest is None or longest_len < 200:
break
longest["content"] = longest["content"][: max(80, longest_len // 2)] + _TRUNCATED.strip()
encoded = json.dumps(body, ensure_ascii=False, indent=2)
if len(encoded) > _MAX_FIELD_CHARS:
body["messages"] = [{"role": "user", "content": _TRUNCATED.strip()}]
encoded = json.dumps(body, ensure_ascii=False, indent=2)
return encoded
def _request_text(payload: Mapping[str, Any], messages: Any) -> str:
tool_names: list[str] = []
tools = payload.get("tools")
if isinstance(tools, list):
for tool in tools:
function = tool.get("function") if isinstance(tool, Mapping) else None
name = function.get("name") if isinstance(function, Mapping) else None
if isinstance(name, str) and name and name not in tool_names:
tool_names.append(name)
lines = [
f"enable_thinking: {_thinking_flag(payload)}",
f"tools: {', '.join(tool_names) if tool_names else '(无)'}",
"",
]
if isinstance(messages, list):
for message in messages:
if not isinstance(message, Mapping):
continue
role = str(message.get("role") or "unknown")
lines.append(f"[{role}]")
lines.append(_content_to_text(message.get("content")))
lines.append("")
return "\n".join(lines).rstrip()
def _append_channel(parts: list[str], incoming: str) -> None:
if not incoming:
return
current = "".join(parts)
if current and (incoming == current or incoming.startswith(current)):
parts.clear()
parts.append(incoming)
return
if current and current.endswith(incoming):
return
parts.append(incoming)
def _preview(text: str) -> str:
compact = " ".join(text.split())
if len(compact) <= _PREVIEW_CHARS:
return compact
return compact[: _PREVIEW_CHARS - 1] + "…"
def _cap(text: str) -> str:
if len(text) <= _MAX_FIELD_CHARS:
return text
return text[: _MAX_FIELD_CHARS - len(_TRUNCATED)] + _TRUNCATED
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,140 @@
"""Normalize OpenAI-compatible thinking fields across provider versions.
Older vLLM / DeepSeek put thinking in ``reasoning_content``. Newer vLLM
reasoning parsers put it in ``reasoning``. DeerFlow's UI and downstream code
canonicalize on ``AIMessage.additional_kwargs.reasoning_content``.
Only fields that actually appeared on a response are echoed back on later
turns, so models that still use ``reasoning_content`` (or send neither) are
unchanged.
"""
from __future__ import annotations
import json
from typing import Any
from langchain_core.messages import AIMessage, AIMessageChunk
def reasoning_to_text(reasoning: Any) -> str:
"""Best-effort extraction of readable reasoning text from provider payloads."""
if isinstance(reasoning, str):
return reasoning
if isinstance(reasoning, list):
parts = [reasoning_to_text(item) for item in reasoning]
return "".join(part for part in parts if part)
if isinstance(reasoning, dict):
for key in ("text", "content", "reasoning"):
value = reasoning.get(key)
if isinstance(value, str):
return value
if value is not None:
text = reasoning_to_text(value)
if text:
return text
try:
return json.dumps(reasoning, ensure_ascii=False)
except TypeError:
return str(reasoning)
if reasoning is None:
return ""
try:
return json.dumps(reasoning, ensure_ascii=False)
except TypeError:
return str(reasoning)
def extract_reasoning_text(payload: dict | None) -> str | None:
"""Read thinking text from ``reasoning_content`` or ``reasoning``.
Prefer ``reasoning_content`` when both are present so older DeepSeek /
vLLM payloads keep their original string. Newer vLLM that only sends
``reasoning`` (string or structured) is still captured.
"""
if not isinstance(payload, dict):
return None
for key in ("reasoning_content", "reasoning"):
value = payload.get(key)
if value is None:
continue
text = reasoning_to_text(value)
if text:
return text
return None
def _merge_streamed_value(existing: Any, incoming: Any) -> Any:
if isinstance(existing, str) and isinstance(incoming, str):
if incoming and incoming not in existing:
return f"{existing}{incoming}"
return existing or incoming
return incoming if incoming is not None else existing
def apply_reasoning_from_payload(
message: AIMessage | AIMessageChunk,
payload: dict | None,
*,
append: bool = True,
) -> AIMessage | AIMessageChunk:
"""Copy provider thinking fields onto ``additional_kwargs`` for UI + echo-back."""
if not isinstance(payload, dict):
return message
additional_kwargs = dict(message.additional_kwargs)
changed = False
if "reasoning" in payload and payload.get("reasoning") is not None:
incoming = payload.get("reasoning")
existing = additional_kwargs.get("reasoning")
merged = _merge_streamed_value(existing, incoming) if append else incoming
if merged is not None and merged != existing:
additional_kwargs["reasoning"] = merged
changed = True
text = extract_reasoning_text(payload)
if text:
existing = additional_kwargs.get("reasoning_content")
if append and isinstance(existing, str) and existing:
if text not in existing:
additional_kwargs["reasoning_content"] = f"{existing}{text}"
changed = True
elif text != existing:
additional_kwargs["reasoning_content"] = text
changed = True
if not changed:
return message
return message.model_copy(update={"additional_kwargs": additional_kwargs})
def fill_ui_reasoning_content(message: AIMessage | AIMessageChunk) -> AIMessage | AIMessageChunk:
"""If LangChain stored ``reasoning`` but not the UI field, copy the text across."""
kwargs = message.additional_kwargs
reasoning = kwargs.get("reasoning")
if reasoning is None or kwargs.get("reasoning_content"):
return message
return apply_reasoning_from_payload(message, {"reasoning": reasoning}, append=False)
def restore_reasoning_fields(payload_msg: dict, orig_msg: AIMessage) -> None:
"""Re-inject the thinking field this assistant turn originally carried.
Newer vLLM stores thinking on ``reasoning`` and we also copy a UI-facing
``reasoning_content`` string from it. Echo the wire field that actually
arrived: ``reasoning`` wins when present so we do not send a synthesized
``reasoning_content`` to a parser that no longer accepts it. Older
DeepSeek / vLLM turns only have ``reasoning_content`` and stay on that
field.
"""
reasoning = orig_msg.additional_kwargs.get("reasoning")
if reasoning is not None:
payload_msg["reasoning"] = reasoning
return
reasoning_content = orig_msg.additional_kwargs.get("reasoning_content")
if reasoning_content is not None:
payload_msg["reasoning_content"] = reasoning_content
@@ -0,0 +1,260 @@
"""Concise model connection and latency diagnostics for production logs."""
from __future__ import annotations
import asyncio
import atexit
import logging
import queue
import threading
import time
from dataclasses import dataclass
from typing import Any
from urllib.parse import urlsplit
from uuid import UUID
from langchain_core.callbacks import BaseCallbackHandler
logger = logging.getLogger(__name__)
class _TimingLogDispatcher:
"""Keep slow log handlers off both the event loop and its tool executor.
Only already-sanitized, small lifecycle records enter this bounded queue.
A daemon thread invokes the existing handlers; producers never wait for
their I/O. Full queues drop diagnostics and report the count when the sink
recovers, instead of retaining unbounded records or stalling generation.
"""
def __init__(self, *, capacity: int = 4096) -> None:
self._queue: queue.Queue[logging.LogRecord] = queue.Queue(maxsize=capacity)
self._lock = threading.Lock()
self._thread: threading.Thread | None = None
self._closed = False
self._dropped = 0
def emit(self, level: int, message: str, *args: Any) -> None:
if not logger.isEnabledFor(level):
return
# Preserve the event's wall-clock timestamp even if delivery is delayed.
# Formatting, filters and handler I/O run only on the consumer thread.
record = logger.makeRecord(logger.name, level, __file__, 0, message, args, None)
with self._lock:
if self._closed:
return
try:
self._queue.put_nowait(record)
except queue.Full:
self._dropped += 1
return
if self._thread is None:
self._thread = threading.Thread(target=self._run, name="deerflow-model-timing-log", daemon=True)
self._thread.start()
def _run(self) -> None:
while True:
try:
record = self._queue.get(timeout=0.1)
except queue.Empty:
with self._lock:
if self._closed:
return
continue
try:
logger.handle(record)
except Exception:
# Diagnostic sinks must not fail a model call or kill the worker.
pass
finally:
self._queue.task_done()
with self._lock:
dropped, self._dropped = self._dropped, 0
if dropped:
try:
logger.warning("LLM_TIMING_LOG_DROPPED count=%d reason=queue_full", dropped)
except Exception:
pass
def close(self, timeout: float = 0.5) -> None:
"""Best-effort drain without making process exit wait on a log sink."""
with self._lock:
self._closed = True
thread = self._thread
if thread is not None and thread is not threading.current_thread():
thread.join(timeout=timeout)
_timing_logs = _TimingLogDispatcher()
atexit.register(_timing_logs.close)
def _safe_base_url(value: str | None) -> str:
"""Log only the upstream origin, never credentials, query, or URL path."""
if not value:
return "-"
try:
parsed = urlsplit(value)
except (TypeError, ValueError):
return "-"
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
return "-"
host = parsed.hostname
try:
port = parsed.port
except ValueError:
return "-"
return f"{parsed.scheme}://{host}:{port}" if port is not None else f"{parsed.scheme}://{host}"
@dataclass(slots=True)
class _CallTiming:
started: float
first_chunk: float | None = None
class ModelTimingCallback(BaseCallbackHandler):
"""Optionally record timing inline and deliver logs on a dedicated queue."""
# LangChain otherwise sends each synchronous callback, including every
# token's no-op after the first, through the shared tool thread pool.
# When enabled, inline work is limited to timestamps, short locked state
# changes and a non-blocking enqueue, never synchronous handler I/O.
run_inline = False
def __init__(self, *, model_name: str, base_url: str | None) -> None:
from deerflow.config.runtime_performance_config import get_runtime_performance_config
self.run_inline = get_runtime_performance_config().model_timing_nonblocking_enabled
self._model_name = model_name
self._base_url = _safe_base_url(base_url)
self._calls: dict[str, _CallTiming] = {}
self._lock = threading.Lock()
def _log(self, level: int, message: str, *args: Any) -> None:
if self.run_inline:
_timing_logs.emit(level, message, *args)
else:
# The default-off path retains LangChain's prior callback dispatch
# and synchronous logging behavior for deployment compatibility.
logger.log(level, message, *args)
def on_chat_model_start(
self,
serialized: dict[str, Any],
messages: list[list[Any]],
*,
run_id: UUID,
**kwargs: Any,
) -> None:
rid = str(run_id)
now = time.monotonic()
with self._lock:
self._calls[rid] = _CallTiming(started=now)
self._log(
logging.INFO,
"LLM_CONNECT_START model=%s base_url=%s run_id=%s message_batches=%d message_count=%d",
self._model_name,
self._base_url,
rid,
len(messages),
sum(len(batch) for batch in messages),
)
def on_llm_new_token(
self,
token: str,
*,
run_id: UUID,
chunk: Any | None = None,
**kwargs: Any,
) -> None:
"""Treat the first callback as connection success, including reasoning-only chunks."""
rid = str(run_id)
now = time.monotonic()
with self._lock:
timing = self._calls.get(rid)
if timing is None or timing.first_chunk is not None:
return
timing.first_chunk = now
first_chunk_ms = int((now - timing.started) * 1000)
self._log(
logging.INFO,
"LLM_CONNECT_OK model=%s base_url=%s run_id=%s first_chunk_ms=%d",
self._model_name,
self._base_url,
rid,
first_chunk_ms,
)
def on_llm_end(self, response: Any, *, run_id: UUID, **kwargs: Any) -> None:
self._finish(run_id=run_id, error=None)
def on_llm_error(self, error: BaseException, *, run_id: UUID, **kwargs: Any) -> None:
self._finish(run_id=run_id, error=error)
def _finish(self, *, run_id: UUID, error: BaseException | None) -> None:
rid = str(run_id)
now = time.monotonic()
cancelled = isinstance(error, (asyncio.CancelledError, GeneratorExit))
with self._lock:
timing = self._calls.pop(rid, None)
if timing is None:
if cancelled:
self._log(
logging.INFO,
"LLM_REQUEST_CANCELLED model=%s base_url=%s run_id=%s status=cancelled error_type=%s",
self._model_name,
self._base_url,
rid,
type(error).__name__,
)
return
self._log(
logging.ERROR,
"LLM_REQUEST_FAILED model=%s base_url=%s run_id=%s error_type=%s",
self._model_name,
self._base_url,
rid,
type(error).__name__ if error else "UnknownLifecycle",
)
return
total_ms = int((now - timing.started) * 1000)
first_chunk_ms = int((timing.first_chunk - timing.started) * 1000) if timing.first_chunk else None
if error is None:
self._log(
logging.INFO,
"LLM_REQUEST_DONE model=%s base_url=%s run_id=%s status=success first_chunk_ms=%s total_ms=%d",
self._model_name,
self._base_url,
rid,
first_chunk_ms if first_chunk_ms is not None else "none",
total_ms,
)
return
if cancelled:
self._log(
logging.INFO,
"LLM_REQUEST_CANCELLED model=%s base_url=%s run_id=%s status=cancelled first_chunk_ms=%s total_ms=%d error_type=%s",
self._model_name,
self._base_url,
rid,
first_chunk_ms if first_chunk_ms is not None else "none",
total_ms,
type(error).__name__,
)
return
self._log(
logging.ERROR,
"LLM_REQUEST_FAILED model=%s base_url=%s run_id=%s status=error first_chunk_ms=%s total_ms=%d error_type=%s",
self._model_name,
self._base_url,
rid,
first_chunk_ms if first_chunk_ms is not None else "none",
total_ms,
type(error).__name__,
)
@@ -6,15 +6,16 @@ from assistant messages and streaming deltas. That breaks interleaved
thinking/tool-call flows because vLLM expects the assistant's prior reasoning to
be echoed back on subsequent turns.
This provider preserves ``reasoning`` on:
- non-streaming responses
- streaming deltas
- multi-turn request payloads
This provider preserves ``reasoning`` (newer vLLM) and
``reasoning_content`` (older vLLM / DeepSeek-style parsers) on:
- non-streaming responses
- streaming deltas
- multi-turn request payloads
UI code still reads the canonical ``additional_kwargs.reasoning_content``.
"""
from __future__ import annotations
import json
from collections.abc import Mapping
from typing import Any, cast
@@ -32,11 +33,12 @@ from langchain_core.messages import (
)
from langchain_core.messages.tool import tool_call_chunk
from langchain_core.outputs import ChatGeneration, ChatGenerationChunk, ChatResult
from langchain_openai import ChatOpenAI
from langchain_openai.chat_models.base import _create_usage_metadata
from deerflow.models.message_ordering import normalize_chat_payload_messages
from langchain_openai import ChatOpenAI
from langchain_openai.chat_models.base import _create_usage_metadata
from deerflow.models.message_ordering import normalize_chat_payload_messages
from deerflow.models.reasoning_fields import apply_reasoning_from_payload, restore_reasoning_fields
def _normalize_vllm_chat_template_kwargs(payload: dict[str, Any]) -> None:
"""Map DeerFlow's legacy ``thinking`` toggle to vLLM/Qwen's ``enable_thinking``.
@@ -66,31 +68,9 @@ def _normalize_vllm_chat_template_kwargs(payload: dict[str, Any]) -> None:
def _reasoning_to_text(reasoning: Any) -> str:
"""Best-effort extraction of readable reasoning text from vLLM payloads."""
if isinstance(reasoning, str):
return reasoning
from deerflow.models.reasoning_fields import reasoning_to_text
if isinstance(reasoning, list):
parts = [_reasoning_to_text(item) for item in reasoning]
return "".join(part for part in parts if part)
if isinstance(reasoning, dict):
for key in ("text", "content", "reasoning"):
value = reasoning.get(key)
if isinstance(value, str):
return value
if value is not None:
text = _reasoning_to_text(value)
if text:
return text
try:
return json.dumps(reasoning, ensure_ascii=False)
except TypeError:
return str(reasoning)
try:
return json.dumps(reasoning, ensure_ascii=False)
except TypeError:
return str(reasoning)
return reasoning_to_text(reasoning)
def _convert_delta_to_message_chunk_with_reasoning(_dict: Mapping[str, Any], default_class: type[BaseMessageChunk]) -> BaseMessageChunk:
@@ -106,12 +86,8 @@ def _convert_delta_to_message_chunk_with_reasoning(_dict: Mapping[str, Any], def
function_call["name"] = ""
additional_kwargs["function_call"] = function_call
reasoning = _dict.get("reasoning")
if reasoning is not None:
additional_kwargs["reasoning"] = reasoning
reasoning_text = _reasoning_to_text(reasoning)
if reasoning_text:
additional_kwargs["reasoning_content"] = reasoning_text
seed = AIMessageChunk(content="", additional_kwargs=additional_kwargs)
additional_kwargs = dict(apply_reasoning_from_payload(seed, dict(_dict)).additional_kwargs)
tool_call_chunks = []
if raw_tool_calls := _dict.get("tool_calls"):
@@ -151,11 +127,7 @@ def _convert_delta_to_message_chunk_with_reasoning(_dict: Mapping[str, Any], def
def _restore_reasoning_field(payload_msg: dict[str, Any], orig_msg: AIMessage) -> None:
"""Re-inject vLLM reasoning onto outgoing assistant messages."""
reasoning = orig_msg.additional_kwargs.get("reasoning")
if reasoning is None:
reasoning = orig_msg.additional_kwargs.get("reasoning_content")
if reasoning is not None:
payload_msg["reasoning"] = reasoning
restore_reasoning_fields(payload_msg, orig_msg)
class VllmChatModel(ChatOpenAI):
@@ -184,14 +156,14 @@ class VllmChatModel(ChatOpenAI):
for payload_msg, orig_msg in zip(payload_messages, original_messages):
if payload_msg.get("role") == "assistant" and isinstance(orig_msg, AIMessage):
_restore_reasoning_field(payload_msg, orig_msg)
else:
ai_messages = [message for message in original_messages if isinstance(message, AIMessage)]
assistant_payloads = [message for message in payload_messages if message.get("role") == "assistant"]
for payload_msg, ai_msg in zip(assistant_payloads, ai_messages):
_restore_reasoning_field(payload_msg, ai_msg)
normalize_chat_payload_messages(payload)
return payload
else:
ai_messages = [message for message in original_messages if isinstance(message, AIMessage)]
assistant_payloads = [message for message in payload_messages if message.get("role") == "assistant"]
for payload_msg, ai_msg in zip(assistant_payloads, ai_messages):
_restore_reasoning_field(payload_msg, ai_msg)
normalize_chat_payload_messages(payload)
return payload
def _create_chat_result(self, response: dict | openai.BaseModel, generation_info: dict | None = None) -> ChatResult:
"""Preserve vLLM reasoning on non-streaming responses."""
@@ -204,13 +176,12 @@ class VllmChatModel(ChatOpenAI):
message = generation.message
if not isinstance(message, AIMessage):
continue
reasoning = choice.get("message", {}).get("reasoning")
if reasoning is None:
continue
message.additional_kwargs["reasoning"] = reasoning
reasoning_text = _reasoning_to_text(reasoning)
if reasoning_text:
message.additional_kwargs["reasoning_content"] = reasoning_text
choice_message = choice.get("message", {}) if isinstance(choice, dict) else {}
generation.message = apply_reasoning_from_payload(
message,
choice_message if isinstance(choice_message, dict) else None,
append=False,
)
return result