feat(turbo): add direct LLM chat mode with runtime toggle
Add 极速 mode that streams via /api/direct-chat without the lead agent stack, plus VITE_TURBO_MODE_VISIBLE in runtime-config.js (default off). Also disable global memory, tighten Qwen/vLLM thinking control for flash mode, and force thinking off on the direct-chat endpoint. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
4f7f07504a
commit
d765afe2ac
23 files changed
+1295
-394
No files matched your search
@@ -31,6 +31,96 @@ def _vllm_disable_chat_template_kwargs(chat_template_kwargs: dict) -> dict:
|
||||
return disable_kwargs
|
||||
|
||||
|
||||
# Qwen3.8 chat templates only accept these three effort names.
|
||||
_QWEN_REASONING_EFFORT = {
|
||||
"minimal": "low",
|
||||
"low": "low",
|
||||
"medium": "medium",
|
||||
"high": "xhigh",
|
||||
"xhigh": "xhigh",
|
||||
}
|
||||
|
||||
_OPENAI_THINKING_OFF_EXTRA_BODY = {
|
||||
"enable_thinking": False,
|
||||
"chat_template_kwargs": {"enable_thinking": False},
|
||||
"thinking": {"type": "disabled"},
|
||||
}
|
||||
|
||||
|
||||
def _is_vllm_provider(model_use: str) -> bool:
|
||||
lowered = (model_use or "").lower()
|
||||
return "vllm_provider" in lowered or lowered.endswith(":vllmchatmodel")
|
||||
|
||||
|
||||
def _is_qwen38_model(model_config) -> bool:
|
||||
blob = f"{getattr(model_config, 'name', '')} {getattr(model_config, 'model', '')}".lower()
|
||||
return "qwen3.8" in blob
|
||||
|
||||
|
||||
def _should_control_openai_thinking(model_config) -> bool:
|
||||
"""Whether this OpenAI-compatible model needs explicit thinking on/off flags.
|
||||
|
||||
Qwen3.8 (and current vLLM reasoning parsers) default to thinking at ``xhigh``.
|
||||
If DeerFlow does not send ``enable_thinking: false``, the first visible answer
|
||||
token waits until that hidden reasoning finishes — third-party chat UIs that
|
||||
disable thinking or stream the reasoning field therefore feel much faster.
|
||||
"""
|
||||
if getattr(model_config, "supports_thinking", False):
|
||||
return True
|
||||
if _is_vllm_provider(getattr(model_config, "use", "")):
|
||||
return True
|
||||
return _is_qwen38_model(model_config)
|
||||
|
||||
|
||||
def _map_qwen_reasoning_effort(effort: object) -> str | None:
|
||||
if effort is None:
|
||||
return None
|
||||
return _QWEN_REASONING_EFFORT.get(str(effort).strip().lower())
|
||||
|
||||
|
||||
def _apply_openai_thinking_control(
|
||||
model_config,
|
||||
model_class,
|
||||
model_settings_from_config: dict,
|
||||
*,
|
||||
thinking_enabled: bool,
|
||||
reasoning_effort: object,
|
||||
) -> None:
|
||||
"""Inject vLLM/Qwen thinking flags when the UI toggle is on or off."""
|
||||
if "extra_body" not in getattr(model_class, "model_fields", {}):
|
||||
return
|
||||
qwen_vllm = _is_vllm_provider(getattr(model_config, "use", "")) or _is_qwen38_model(model_config)
|
||||
if thinking_enabled:
|
||||
if not qwen_vllm:
|
||||
return
|
||||
enable_body: dict = {
|
||||
"enable_thinking": True,
|
||||
"chat_template_kwargs": {"enable_thinking": True},
|
||||
}
|
||||
mapped_effort = _map_qwen_reasoning_effort(reasoning_effort)
|
||||
if mapped_effort:
|
||||
enable_body["chat_template_kwargs"]["reasoning_effort"] = mapped_effort
|
||||
model_settings_from_config["extra_body"] = _deep_merge_dicts(
|
||||
model_settings_from_config.get("extra_body"),
|
||||
enable_body,
|
||||
)
|
||||
return
|
||||
if not _should_control_openai_thinking(model_config):
|
||||
return
|
||||
disable_body = (
|
||||
{
|
||||
"enable_thinking": False,
|
||||
"chat_template_kwargs": {"enable_thinking": False},
|
||||
}
|
||||
if qwen_vllm
|
||||
else _OPENAI_THINKING_OFF_EXTRA_BODY
|
||||
)
|
||||
model_settings_from_config["extra_body"] = _deep_merge_dicts(
|
||||
model_settings_from_config.get("extra_body"),
|
||||
disable_body,
|
||||
)
|
||||
|
||||
|
||||
def _enable_stream_usage_by_default(model_use_path: str, model_settings_from_config: dict) -> None:
|
||||
"""Enable stream usage for OpenAI-compatible models unless explicitly configured.
|
||||
|
||||
@@ -127,10 +217,19 @@ def create_chat_model(name: str | None = None, thinking_enabled: bool = False, *
|
||||
elif has_thinking_settings and effective_wte.get("thinking", {}).get("type"):
|
||||
# Native langchain_anthropic: thinking is a direct constructor parameter
|
||||
model_settings_from_config["thinking"] = {"type": "disabled"}
|
||||
requested_reasoning_effort = kwargs.get("reasoning_effort")
|
||||
if not model_config.supports_reasoning_effort:
|
||||
kwargs.pop("reasoning_effort", None)
|
||||
model_settings_from_config.pop("reasoning_effort", None)
|
||||
|
||||
_apply_openai_thinking_control(
|
||||
model_config,
|
||||
model_class,
|
||||
model_settings_from_config,
|
||||
thinking_enabled=thinking_enabled,
|
||||
reasoning_effort=requested_reasoning_effort,
|
||||
)
|
||||
|
||||
# Hard override: force-disable thinking regardless of the per-model config shape.
|
||||
# Injected only for OpenAI-compatible classes (those declaring `extra_body`);
|
||||
# other providers would reject the kwarg, so they are skipped (they ignore it).
|
||||
@@ -139,15 +238,15 @@ def create_chat_model(name: str | None = None, thinking_enabled: bool = False, *
|
||||
# (DeepSeek-V3.1+/Qwen3/GLM hybrid models on api.siliconflow.cn — the knob the
|
||||
# nested ``chat_template_kwargs`` shape does NOT reach), vLLM/Qwen native
|
||||
# ``chat_template_kwargs.enable_thinking``, and Zhipu GLM ``thinking.type``.
|
||||
# Only models that explicitly opt into thinking should receive vendor-specific
|
||||
# disable payloads. Generic OpenAI-compatible gateways (including LiteLLM)
|
||||
# may reject these otherwise-unknown fields instead of forwarding them.
|
||||
if (
|
||||
force_disable_thinking
|
||||
and not thinking_enabled
|
||||
and model_config.supports_thinking
|
||||
and "extra_body" in getattr(model_class, "model_fields", {})
|
||||
):
|
||||
# Only models that explicitly opt into thinking should receive vendor-specific
|
||||
# disable payloads. Generic OpenAI-compatible gateways (including LiteLLM)
|
||||
# may reject these otherwise-unknown fields instead of forwarding them.
|
||||
if (
|
||||
force_disable_thinking
|
||||
and not thinking_enabled
|
||||
and model_config.supports_thinking
|
||||
and "extra_body" in getattr(model_class, "model_fields", {})
|
||||
):
|
||||
model_settings_from_config["extra_body"] = _deep_merge_dicts(
|
||||
model_settings_from_config.get("extra_body"),
|
||||
{"enable_thinking": False, "chat_template_kwargs": {"enable_thinking": False}, "thinking": {"type": "disabled"}},
|
||||
|
||||
+27
-17
@@ -29,11 +29,11 @@ from typing import Any
|
||||
|
||||
from langchain_core.language_models import LanguageModelInput
|
||||
from langchain_core.messages import AIMessage, AIMessageChunk
|
||||
from langchain_core.outputs import ChatGeneration, ChatGenerationChunk, ChatResult
|
||||
from langchain_openai import ChatOpenAI
|
||||
|
||||
from deerflow.models.message_ordering import normalize_chat_payload_messages
|
||||
|
||||
from langchain_core.outputs import ChatGeneration, ChatGenerationChunk, ChatResult
|
||||
from langchain_openai import ChatOpenAI
|
||||
|
||||
from deerflow.models.message_ordering import normalize_chat_payload_messages
|
||||
|
||||
|
||||
class PatchedChatOpenAI(ChatOpenAI):
|
||||
"""ChatOpenAI with ``thought_signature`` preservation for Gemini thinking via OpenAI gateway.
|
||||
@@ -89,16 +89,16 @@ class PatchedChatOpenAI(ChatOpenAI):
|
||||
if payload_msg.get("role") == "assistant" and isinstance(orig_msg, AIMessage):
|
||||
_restore_tool_call_signatures(payload_msg, orig_msg)
|
||||
_restore_reasoning_content(payload_msg, orig_msg)
|
||||
else:
|
||||
# Fallback: match assistant-role entries positionally against AIMessages.
|
||||
ai_messages = [m for m in original_messages if isinstance(m, AIMessage)]
|
||||
assistant_payloads = [(i, m) for i, m in enumerate(payload_messages) if m.get("role") == "assistant"]
|
||||
for (_, payload_msg), ai_msg in zip(assistant_payloads, ai_messages):
|
||||
_restore_tool_call_signatures(payload_msg, ai_msg)
|
||||
_restore_reasoning_content(payload_msg, ai_msg)
|
||||
|
||||
normalize_chat_payload_messages(payload)
|
||||
return payload
|
||||
else:
|
||||
# Fallback: match assistant-role entries positionally against AIMessages.
|
||||
ai_messages = [m for m in original_messages if isinstance(m, AIMessage)]
|
||||
assistant_payloads = [(i, m) for i, m in enumerate(payload_messages) if m.get("role") == "assistant"]
|
||||
for (_, payload_msg), ai_msg in zip(assistant_payloads, ai_messages):
|
||||
_restore_tool_call_signatures(payload_msg, ai_msg)
|
||||
_restore_reasoning_content(payload_msg, ai_msg)
|
||||
|
||||
normalize_chat_payload_messages(payload)
|
||||
return payload
|
||||
|
||||
def _convert_chunk_to_generation_chunk(
|
||||
self,
|
||||
@@ -208,11 +208,21 @@ def _restore_reasoning_content(payload_msg: dict, orig_msg: AIMessage) -> None:
|
||||
|
||||
|
||||
def _extract_reasoning_content(payload: dict | None) -> str | None:
|
||||
"""Extract reasoning text from provider payload delta/message objects."""
|
||||
"""Extract reasoning text from provider payload delta/message objects.
|
||||
|
||||
vLLM Qwen reasoning parsers use ``reasoning_content`` on streaming deltas
|
||||
and ``reasoning`` on some non-stream / older builds. Read both so the UI
|
||||
can show the first thinking token instead of waiting for answer content.
|
||||
"""
|
||||
if not isinstance(payload, dict):
|
||||
return None
|
||||
reasoning_content = payload.get("reasoning_content")
|
||||
return reasoning_content if isinstance(reasoning_content, str) and reasoning_content else None
|
||||
if isinstance(reasoning_content, str) and reasoning_content:
|
||||
return reasoning_content
|
||||
reasoning = payload.get("reasoning")
|
||||
if isinstance(reasoning, str) and reasoning:
|
||||
return reasoning
|
||||
return None
|
||||
|
||||
|
||||
def _append_reasoning_content(message: AIMessage | AIMessageChunk, reasoning_content: str) -> AIMessage | AIMessageChunk:
|
||||
|
||||
+15
-13
@@ -32,11 +32,11 @@ from langchain_core.messages import (
|
||||
)
|
||||
from langchain_core.messages.tool import tool_call_chunk
|
||||
from langchain_core.outputs import ChatGeneration, ChatGenerationChunk, ChatResult
|
||||
from langchain_openai import ChatOpenAI
|
||||
from langchain_openai.chat_models.base import _create_usage_metadata
|
||||
|
||||
from deerflow.models.message_ordering import normalize_chat_payload_messages
|
||||
|
||||
from langchain_openai import ChatOpenAI
|
||||
from langchain_openai.chat_models.base import _create_usage_metadata
|
||||
|
||||
from deerflow.models.message_ordering import normalize_chat_payload_messages
|
||||
|
||||
|
||||
def _normalize_vllm_chat_template_kwargs(payload: dict[str, Any]) -> None:
|
||||
"""Map DeerFlow's legacy ``thinking`` toggle to vLLM/Qwen's ``enable_thinking``.
|
||||
@@ -107,6 +107,8 @@ def _convert_delta_to_message_chunk_with_reasoning(_dict: Mapping[str, Any], def
|
||||
additional_kwargs["function_call"] = function_call
|
||||
|
||||
reasoning = _dict.get("reasoning")
|
||||
if reasoning is None:
|
||||
reasoning = _dict.get("reasoning_content")
|
||||
if reasoning is not None:
|
||||
additional_kwargs["reasoning"] = reasoning
|
||||
reasoning_text = _reasoning_to_text(reasoning)
|
||||
@@ -184,14 +186,14 @@ class VllmChatModel(ChatOpenAI):
|
||||
for payload_msg, orig_msg in zip(payload_messages, original_messages):
|
||||
if payload_msg.get("role") == "assistant" and isinstance(orig_msg, AIMessage):
|
||||
_restore_reasoning_field(payload_msg, orig_msg)
|
||||
else:
|
||||
ai_messages = [message for message in original_messages if isinstance(message, AIMessage)]
|
||||
assistant_payloads = [message for message in payload_messages if message.get("role") == "assistant"]
|
||||
for payload_msg, ai_msg in zip(assistant_payloads, ai_messages):
|
||||
_restore_reasoning_field(payload_msg, ai_msg)
|
||||
|
||||
normalize_chat_payload_messages(payload)
|
||||
return payload
|
||||
else:
|
||||
ai_messages = [message for message in original_messages if isinstance(message, AIMessage)]
|
||||
assistant_payloads = [message for message in payload_messages if message.get("role") == "assistant"]
|
||||
for payload_msg, ai_msg in zip(assistant_payloads, ai_messages):
|
||||
_restore_reasoning_field(payload_msg, ai_msg)
|
||||
|
||||
normalize_chat_payload_messages(payload)
|
||||
return payload
|
||||
|
||||
def _create_chat_result(self, response: dict | openai.BaseModel, generation_info: dict | None = None) -> ChatResult:
|
||||
"""Preserve vLLM reasoning on non-streaming responses."""
|
||||
|
||||
Reference in new issue
Block a user