fix: accept vLLM thinking in reasoning or reasoning_content
Newer vLLM parsers put CoT in reasoning; older builds and DeepSeek still use reasoning_content. Canonicalize to the UI field and echo only the original wire field so other ChatOpenAI models stay unchanged. Point qwen3.6 at VllmChatModel. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
d765afe2ac
commit
f4c7d41a60
10 files changed
+371
-123
No files matched your search
+26
-40
@@ -33,6 +33,7 @@ from langchain_core.outputs import ChatGeneration, ChatGenerationChunk, ChatResu
|
||||
from langchain_openai import ChatOpenAI
|
||||
|
||||
from deerflow.models.message_ordering import normalize_chat_payload_messages
|
||||
from deerflow.models.reasoning_fields import apply_reasoning_from_payload, extract_reasoning_text, fill_ui_reasoning_content, restore_reasoning_fields
|
||||
|
||||
|
||||
class PatchedChatOpenAI(ChatOpenAI):
|
||||
@@ -118,12 +119,12 @@ class PatchedChatOpenAI(ChatOpenAI):
|
||||
choices = chunk.get("choices", []) or chunk.get("chunk", {}).get("choices", [])
|
||||
choice = choices[0] if choices else {}
|
||||
delta = choice.get("delta") if isinstance(choice, dict) else None
|
||||
reasoning_content = _extract_reasoning_content(delta if isinstance(delta, dict) else None)
|
||||
if reasoning_content and isinstance(generation_chunk.message, AIMessageChunk):
|
||||
generation_chunk.message = _append_reasoning_content(
|
||||
if isinstance(generation_chunk.message, AIMessageChunk):
|
||||
generation_chunk.message = apply_reasoning_from_payload(
|
||||
generation_chunk.message,
|
||||
reasoning_content,
|
||||
delta if isinstance(delta, dict) else None,
|
||||
)
|
||||
generation_chunk.message = fill_ui_reasoning_content(generation_chunk.message)
|
||||
return generation_chunk
|
||||
|
||||
def _create_chat_result(
|
||||
@@ -141,13 +142,19 @@ class PatchedChatOpenAI(ChatOpenAI):
|
||||
message = generation.message
|
||||
choice = choices[index] if index < len(choices) else {}
|
||||
choice_message = choice.get("message", {}) if isinstance(choice, dict) else {}
|
||||
reasoning_content = _extract_reasoning_content(choice_message if isinstance(choice_message, dict) else None)
|
||||
if reasoning_content and isinstance(message, AIMessage):
|
||||
message = _append_reasoning_content(message, reasoning_content)
|
||||
generation = ChatGeneration(
|
||||
message=message,
|
||||
generation_info=generation.generation_info,
|
||||
if isinstance(message, AIMessage):
|
||||
updated = apply_reasoning_from_payload(
|
||||
message,
|
||||
choice_message if isinstance(choice_message, dict) else None,
|
||||
append=False,
|
||||
)
|
||||
updated = fill_ui_reasoning_content(updated)
|
||||
if updated is not message:
|
||||
message = updated
|
||||
generation = ChatGeneration(
|
||||
message=message,
|
||||
generation_info=generation.generation_info,
|
||||
)
|
||||
generations.append(generation)
|
||||
|
||||
return ChatResult(generations=generations, llm_output=result.llm_output)
|
||||
@@ -195,43 +202,22 @@ def _restore_tool_call_signatures(payload_msg: dict, orig_msg: AIMessage) -> Non
|
||||
|
||||
|
||||
def _restore_reasoning_content(payload_msg: dict, orig_msg: AIMessage) -> None:
|
||||
"""Re-inject ``reasoning_content`` onto assistant payload message.
|
||||
"""Re-inject thinking fields onto assistant payload message.
|
||||
|
||||
Some thinking-enabled OpenAI-compatible providers require each historical
|
||||
assistant turn to include the original ``reasoning_content`` in subsequent
|
||||
requests. LangChain stores it in ``AIMessage.additional_kwargs`` but does
|
||||
not always serialize it back into ``messages``.
|
||||
assistant turn to include the original thinking field in subsequent
|
||||
requests. Newer vLLM puts that text in ``reasoning``; DeepSeek and older
|
||||
vLLM use ``reasoning_content``. Echo only the fields this turn originally
|
||||
carried so other models are not given an extra field.
|
||||
"""
|
||||
reasoning_content = orig_msg.additional_kwargs.get("reasoning_content")
|
||||
if reasoning_content is not None:
|
||||
payload_msg["reasoning_content"] = reasoning_content
|
||||
restore_reasoning_fields(payload_msg, orig_msg)
|
||||
|
||||
|
||||
def _extract_reasoning_content(payload: dict | None) -> str | None:
|
||||
"""Extract reasoning text from provider payload delta/message objects.
|
||||
|
||||
vLLM Qwen reasoning parsers use ``reasoning_content`` on streaming deltas
|
||||
and ``reasoning`` on some non-stream / older builds. Read both so the UI
|
||||
can show the first thinking token instead of waiting for answer content.
|
||||
"""
|
||||
if not isinstance(payload, dict):
|
||||
return None
|
||||
reasoning_content = payload.get("reasoning_content")
|
||||
if isinstance(reasoning_content, str) and reasoning_content:
|
||||
return reasoning_content
|
||||
reasoning = payload.get("reasoning")
|
||||
if isinstance(reasoning, str) and reasoning:
|
||||
return reasoning
|
||||
return None
|
||||
"""Extract reasoning text from provider payload delta/message objects."""
|
||||
return extract_reasoning_text(payload)
|
||||
|
||||
|
||||
def _append_reasoning_content(message: AIMessage | AIMessageChunk, reasoning_content: str) -> AIMessage | AIMessageChunk:
|
||||
"""Append streamed reasoning chunks, dedupe complete response reasoning."""
|
||||
additional_kwargs = dict(message.additional_kwargs)
|
||||
existing = additional_kwargs.get("reasoning_content")
|
||||
if isinstance(existing, str) and existing:
|
||||
if reasoning_content not in existing:
|
||||
additional_kwargs["reasoning_content"] = f"{existing}{reasoning_content}"
|
||||
else:
|
||||
additional_kwargs["reasoning_content"] = reasoning_content
|
||||
return message.model_copy(update={"additional_kwargs": additional_kwargs})
|
||||
return apply_reasoning_from_payload(message, {"reasoning_content": reasoning_content})
|
||||
+140
@@ -0,0 +1,140 @@
|
||||
"""Normalize OpenAI-compatible thinking fields across provider versions.
|
||||
|
||||
Older vLLM / DeepSeek put thinking in ``reasoning_content``. Newer vLLM
|
||||
reasoning parsers put it in ``reasoning``. DeerFlow's UI and downstream code
|
||||
canonicalize on ``AIMessage.additional_kwargs.reasoning_content``.
|
||||
|
||||
Only fields that actually appeared on a response are echoed back on later
|
||||
turns, so models that still use ``reasoning_content`` (or send neither) are
|
||||
unchanged.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from langchain_core.messages import AIMessage, AIMessageChunk
|
||||
|
||||
|
||||
def reasoning_to_text(reasoning: Any) -> str:
|
||||
"""Best-effort extraction of readable reasoning text from provider payloads."""
|
||||
if isinstance(reasoning, str):
|
||||
return reasoning
|
||||
|
||||
if isinstance(reasoning, list):
|
||||
parts = [reasoning_to_text(item) for item in reasoning]
|
||||
return "".join(part for part in parts if part)
|
||||
|
||||
if isinstance(reasoning, dict):
|
||||
for key in ("text", "content", "reasoning"):
|
||||
value = reasoning.get(key)
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
if value is not None:
|
||||
text = reasoning_to_text(value)
|
||||
if text:
|
||||
return text
|
||||
try:
|
||||
return json.dumps(reasoning, ensure_ascii=False)
|
||||
except TypeError:
|
||||
return str(reasoning)
|
||||
|
||||
if reasoning is None:
|
||||
return ""
|
||||
try:
|
||||
return json.dumps(reasoning, ensure_ascii=False)
|
||||
except TypeError:
|
||||
return str(reasoning)
|
||||
|
||||
|
||||
def extract_reasoning_text(payload: dict | None) -> str | None:
|
||||
"""Read thinking text from ``reasoning_content`` or ``reasoning``.
|
||||
|
||||
Prefer ``reasoning_content`` when both are present so older DeepSeek /
|
||||
vLLM payloads keep their original string. Newer vLLM that only sends
|
||||
``reasoning`` (string or structured) is still captured.
|
||||
"""
|
||||
if not isinstance(payload, dict):
|
||||
return None
|
||||
for key in ("reasoning_content", "reasoning"):
|
||||
value = payload.get(key)
|
||||
if value is None:
|
||||
continue
|
||||
text = reasoning_to_text(value)
|
||||
if text:
|
||||
return text
|
||||
return None
|
||||
|
||||
|
||||
def _merge_streamed_value(existing: Any, incoming: Any) -> Any:
|
||||
if isinstance(existing, str) and isinstance(incoming, str):
|
||||
if incoming and incoming not in existing:
|
||||
return f"{existing}{incoming}"
|
||||
return existing or incoming
|
||||
return incoming if incoming is not None else existing
|
||||
|
||||
|
||||
def apply_reasoning_from_payload(
|
||||
message: AIMessage | AIMessageChunk,
|
||||
payload: dict | None,
|
||||
*,
|
||||
append: bool = True,
|
||||
) -> AIMessage | AIMessageChunk:
|
||||
"""Copy provider thinking fields onto ``additional_kwargs`` for UI + echo-back."""
|
||||
if not isinstance(payload, dict):
|
||||
return message
|
||||
|
||||
additional_kwargs = dict(message.additional_kwargs)
|
||||
changed = False
|
||||
|
||||
if "reasoning" in payload and payload.get("reasoning") is not None:
|
||||
incoming = payload.get("reasoning")
|
||||
existing = additional_kwargs.get("reasoning")
|
||||
merged = _merge_streamed_value(existing, incoming) if append else incoming
|
||||
if merged is not None and merged != existing:
|
||||
additional_kwargs["reasoning"] = merged
|
||||
changed = True
|
||||
|
||||
text = extract_reasoning_text(payload)
|
||||
if text:
|
||||
existing = additional_kwargs.get("reasoning_content")
|
||||
if append and isinstance(existing, str) and existing:
|
||||
if text not in existing:
|
||||
additional_kwargs["reasoning_content"] = f"{existing}{text}"
|
||||
changed = True
|
||||
elif text != existing:
|
||||
additional_kwargs["reasoning_content"] = text
|
||||
changed = True
|
||||
|
||||
if not changed:
|
||||
return message
|
||||
return message.model_copy(update={"additional_kwargs": additional_kwargs})
|
||||
|
||||
|
||||
def fill_ui_reasoning_content(message: AIMessage | AIMessageChunk) -> AIMessage | AIMessageChunk:
|
||||
"""If LangChain stored ``reasoning`` but not the UI field, copy the text across."""
|
||||
kwargs = message.additional_kwargs
|
||||
reasoning = kwargs.get("reasoning")
|
||||
if reasoning is None or kwargs.get("reasoning_content"):
|
||||
return message
|
||||
return apply_reasoning_from_payload(message, {"reasoning": reasoning}, append=False)
|
||||
|
||||
|
||||
def restore_reasoning_fields(payload_msg: dict, orig_msg: AIMessage) -> None:
|
||||
"""Re-inject the thinking field this assistant turn originally carried.
|
||||
|
||||
Newer vLLM stores thinking on ``reasoning`` and we also copy a UI-facing
|
||||
``reasoning_content`` string from it. Echo the wire field that actually
|
||||
arrived: ``reasoning`` wins when present so we do not send a synthesized
|
||||
``reasoning_content`` to a parser that no longer accepts it. Older
|
||||
DeepSeek / vLLM turns only have ``reasoning_content`` and stay on that
|
||||
field.
|
||||
"""
|
||||
reasoning = orig_msg.additional_kwargs.get("reasoning")
|
||||
if reasoning is not None:
|
||||
payload_msg["reasoning"] = reasoning
|
||||
return
|
||||
reasoning_content = orig_msg.additional_kwargs.get("reasoning_content")
|
||||
if reasoning_content is not None:
|
||||
payload_msg["reasoning_content"] = reasoning_content
|
||||
+18
-49
@@ -6,15 +6,16 @@ from assistant messages and streaming deltas. That breaks interleaved
|
||||
thinking/tool-call flows because vLLM expects the assistant's prior reasoning to
|
||||
be echoed back on subsequent turns.
|
||||
|
||||
This provider preserves ``reasoning`` on:
|
||||
- non-streaming responses
|
||||
- streaming deltas
|
||||
- multi-turn request payloads
|
||||
This provider preserves ``reasoning`` (newer vLLM) and
|
||||
``reasoning_content`` (older vLLM / DeepSeek-style parsers) on:
|
||||
- non-streaming responses
|
||||
- streaming deltas
|
||||
- multi-turn request payloads
|
||||
UI code still reads the canonical ``additional_kwargs.reasoning_content``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from typing import Any, cast
|
||||
|
||||
@@ -36,6 +37,7 @@ from langchain_openai import ChatOpenAI
|
||||
from langchain_openai.chat_models.base import _create_usage_metadata
|
||||
|
||||
from deerflow.models.message_ordering import normalize_chat_payload_messages
|
||||
from deerflow.models.reasoning_fields import apply_reasoning_from_payload, restore_reasoning_fields
|
||||
|
||||
|
||||
def _normalize_vllm_chat_template_kwargs(payload: dict[str, Any]) -> None:
|
||||
@@ -66,31 +68,9 @@ def _normalize_vllm_chat_template_kwargs(payload: dict[str, Any]) -> None:
|
||||
|
||||
def _reasoning_to_text(reasoning: Any) -> str:
|
||||
"""Best-effort extraction of readable reasoning text from vLLM payloads."""
|
||||
if isinstance(reasoning, str):
|
||||
return reasoning
|
||||
from deerflow.models.reasoning_fields import reasoning_to_text
|
||||
|
||||
if isinstance(reasoning, list):
|
||||
parts = [_reasoning_to_text(item) for item in reasoning]
|
||||
return "".join(part for part in parts if part)
|
||||
|
||||
if isinstance(reasoning, dict):
|
||||
for key in ("text", "content", "reasoning"):
|
||||
value = reasoning.get(key)
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
if value is not None:
|
||||
text = _reasoning_to_text(value)
|
||||
if text:
|
||||
return text
|
||||
try:
|
||||
return json.dumps(reasoning, ensure_ascii=False)
|
||||
except TypeError:
|
||||
return str(reasoning)
|
||||
|
||||
try:
|
||||
return json.dumps(reasoning, ensure_ascii=False)
|
||||
except TypeError:
|
||||
return str(reasoning)
|
||||
return reasoning_to_text(reasoning)
|
||||
|
||||
|
||||
def _convert_delta_to_message_chunk_with_reasoning(_dict: Mapping[str, Any], default_class: type[BaseMessageChunk]) -> BaseMessageChunk:
|
||||
@@ -106,14 +86,8 @@ def _convert_delta_to_message_chunk_with_reasoning(_dict: Mapping[str, Any], def
|
||||
function_call["name"] = ""
|
||||
additional_kwargs["function_call"] = function_call
|
||||
|
||||
reasoning = _dict.get("reasoning")
|
||||
if reasoning is None:
|
||||
reasoning = _dict.get("reasoning_content")
|
||||
if reasoning is not None:
|
||||
additional_kwargs["reasoning"] = reasoning
|
||||
reasoning_text = _reasoning_to_text(reasoning)
|
||||
if reasoning_text:
|
||||
additional_kwargs["reasoning_content"] = reasoning_text
|
||||
seed = AIMessageChunk(content="", additional_kwargs=additional_kwargs)
|
||||
additional_kwargs = dict(apply_reasoning_from_payload(seed, dict(_dict)).additional_kwargs)
|
||||
|
||||
tool_call_chunks = []
|
||||
if raw_tool_calls := _dict.get("tool_calls"):
|
||||
@@ -153,11 +127,7 @@ def _convert_delta_to_message_chunk_with_reasoning(_dict: Mapping[str, Any], def
|
||||
|
||||
def _restore_reasoning_field(payload_msg: dict[str, Any], orig_msg: AIMessage) -> None:
|
||||
"""Re-inject vLLM reasoning onto outgoing assistant messages."""
|
||||
reasoning = orig_msg.additional_kwargs.get("reasoning")
|
||||
if reasoning is None:
|
||||
reasoning = orig_msg.additional_kwargs.get("reasoning_content")
|
||||
if reasoning is not None:
|
||||
payload_msg["reasoning"] = reasoning
|
||||
restore_reasoning_fields(payload_msg, orig_msg)
|
||||
|
||||
|
||||
class VllmChatModel(ChatOpenAI):
|
||||
@@ -206,13 +176,12 @@ class VllmChatModel(ChatOpenAI):
|
||||
message = generation.message
|
||||
if not isinstance(message, AIMessage):
|
||||
continue
|
||||
reasoning = choice.get("message", {}).get("reasoning")
|
||||
if reasoning is None:
|
||||
continue
|
||||
message.additional_kwargs["reasoning"] = reasoning
|
||||
reasoning_text = _reasoning_to_text(reasoning)
|
||||
if reasoning_text:
|
||||
message.additional_kwargs["reasoning_content"] = reasoning_text
|
||||
choice_message = choice.get("message", {}) if isinstance(choice, dict) else {}
|
||||
generation.message = apply_reasoning_from_payload(
|
||||
message,
|
||||
choice_message if isinstance(choice_message, dict) else None,
|
||||
append=False,
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
Reference in new issue
Block a user