sync: 鍚屾灞€鍩熺綉鏈€鏂颁唬鐮?(source master @ b69568e4)
This commit is contained in:
1 parent
4f7f07504a
commit
8bc8b0e082
1651 files changed
+278599
-11416
No files matched your search
+106
@@ -0,0 +1,106 @@
|
||||
"""Invocation-local text and completion checks for compatible chat streams."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
|
||||
def needs_qwen_flash_next_integrity(*model_names: str | None) -> bool:
|
||||
"""Match the affected model id or its local alias, without broad Qwen rules."""
|
||||
return any(
|
||||
name.strip().lower().replace("_", "-").rsplit("/", 1)[-1] == "qwen3.8-flash-next"
|
||||
for name in model_names
|
||||
if isinstance(name, str)
|
||||
)
|
||||
|
||||
|
||||
class IncompleteChatStreamError(RuntimeError):
|
||||
"""A provider stream ended before a complete response could be committed.
|
||||
|
||||
Only a fixed reason is exposed. Provider text, URLs and error bodies must
|
||||
never appear in an exception that the gateway may send to a browser.
|
||||
"""
|
||||
|
||||
def __init__(self, reason: str) -> None:
|
||||
self.reason = reason
|
||||
super().__init__(f"Incomplete OpenAI-compatible chat stream: {reason}")
|
||||
|
||||
|
||||
@dataclass
|
||||
class ChatStreamState:
|
||||
"""Track one request, never a model instance or a persisted conversation."""
|
||||
|
||||
observed_chat_frame: bool = False
|
||||
terminal_received: bool = False
|
||||
content: dict[int, str] = field(default_factory=dict)
|
||||
reasoning: dict[int, str] = field(default_factory=dict)
|
||||
|
||||
def observe(self, choices: list[dict[str, Any]]) -> None:
|
||||
self.observed_chat_frame = True
|
||||
# LangChain consumes the first choice, so another choice's finish
|
||||
# reason must not make an unfinished first choice look complete.
|
||||
if choices and choices[0].get("finish_reason"):
|
||||
self.terminal_received = True
|
||||
|
||||
def ensure_finished(self) -> None:
|
||||
# Responses API and adapter-generated chunks do not pass through the
|
||||
# chat-completions converter and have their own completion semantics.
|
||||
if self.observed_chat_frame and not self.terminal_received:
|
||||
raise IncompleteChatStreamError("missing_terminal_frame")
|
||||
|
||||
def normalize_text(
|
||||
self,
|
||||
*,
|
||||
index: int,
|
||||
delta: dict[str, Any],
|
||||
original_delta: Mapping[str, Any] | None,
|
||||
message: Mapping[str, Any] | None,
|
||||
) -> None:
|
||||
"""Convert explicit full-message snapshots to suffixes.
|
||||
|
||||
A real delta always wins and is appended verbatim, including repeated
|
||||
words. Only text supplied through ``message`` is treated as a full
|
||||
snapshot; prefix guessing on ordinary deltas would lose real tokens.
|
||||
"""
|
||||
for fields, accumulated in (
|
||||
(("content",), self.content),
|
||||
(("reasoning_content", "reasoning"), self.reasoning),
|
||||
):
|
||||
source = original_delta or {}
|
||||
current = accumulated.get(index, "")
|
||||
incoming = _first_text(source, fields)
|
||||
if incoming is not None:
|
||||
accumulated[index] = current + incoming[1]
|
||||
continue
|
||||
# Structured content and vendor reasoning objects are opaque.
|
||||
# Preserve them instead of replacing them with a text snapshot.
|
||||
if any(source.get(name) is not None and not isinstance(source[name], str) for name in fields):
|
||||
continue
|
||||
|
||||
snapshot = _first_text(message or {}, fields)
|
||||
if snapshot is None:
|
||||
continue
|
||||
name, text = snapshot
|
||||
if text.startswith(current):
|
||||
suffix = text[len(current):]
|
||||
accumulated[index] = text
|
||||
elif current.startswith(text):
|
||||
# A duplicate/stale snapshot adds nothing to the live text.
|
||||
suffix = ""
|
||||
else:
|
||||
# A delta cannot retract already-yielded bytes. Retry before
|
||||
# committing contradictory text into the conversation.
|
||||
raise IncompleteChatStreamError("inconsistent_message_snapshot")
|
||||
for alias in fields:
|
||||
delta.pop(alias, None)
|
||||
delta[name] = suffix
|
||||
|
||||
|
||||
def _first_text(source: Mapping[str, Any], fields: tuple[str, ...]) -> tuple[str, str] | None:
|
||||
for name in fields:
|
||||
value = source.get(name)
|
||||
if isinstance(value, str) and value:
|
||||
return name, value
|
||||
return None
|
||||
@@ -0,0 +1,50 @@
|
||||
"""Shared detection for adaptive context recovery (no assumed model window)."""
|
||||
|
||||
from collections.abc import Mapping
|
||||
|
||||
CONTEXT_RECOVERY_MESSAGE = "请补充本次需要处理的具体问题或关键材料,我会继续为你解答。"
|
||||
|
||||
_PATTERNS = (
|
||||
"context_length_exceeded",
|
||||
"maximum context length",
|
||||
"max context length",
|
||||
"context window exceeds limit",
|
||||
"context window exceeded",
|
||||
"exceeds the context window",
|
||||
"max_model_len",
|
||||
"prompt is too long",
|
||||
"input is too long",
|
||||
"too many tokens",
|
||||
"input token count",
|
||||
"prompt token count",
|
||||
"exceeds the model's context",
|
||||
"exceeds model context",
|
||||
"上下文长度",
|
||||
"最大上下文",
|
||||
"输入 token 数超过",
|
||||
"输入token数超过",
|
||||
)
|
||||
|
||||
|
||||
def is_context_length_error(exc: BaseException) -> bool:
|
||||
"""Inspect provider error codes as well as text, without matching all 400s."""
|
||||
values = [str(exc), str(getattr(exc, "code", "")), str(getattr(exc, "error_code", ""))]
|
||||
# Compatible gateways may use numeric codes and put the only context-size
|
||||
# signal in their message. Our adapter keeps that text private so __str__
|
||||
# and framework logs cannot expose provider bodies; classify it in-process.
|
||||
classification_detail = getattr(exc, "_classification_detail", None)
|
||||
if isinstance(classification_detail, str):
|
||||
values.append(classification_detail)
|
||||
body = getattr(exc, "body", None)
|
||||
if isinstance(body, Mapping):
|
||||
error = body.get("error", body)
|
||||
if isinstance(error, Mapping):
|
||||
values.extend(str(error.get(key, "")) for key in ("code", "type", "message"))
|
||||
return any(pattern in value.lower() for value in values for pattern in _PATTERNS)
|
||||
|
||||
|
||||
class ContextRecoveryExhausted(RuntimeError):
|
||||
"""Internal failure for contracts that cannot degrade to conversational text."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__("context_length_exceeded: adaptive recovery exhausted")
|
||||
@@ -2,8 +2,11 @@ import logging
|
||||
|
||||
from langchain.chat_models import BaseChatModel
|
||||
|
||||
from deerflow.config import get_app_config
|
||||
from deerflow.config.app_config import AppConfig
|
||||
from deerflow.config import get_app_config
|
||||
from deerflow.config.app_config import AppConfig
|
||||
from deerflow.config.model_config import WriteFileTransport
|
||||
from deerflow.config.system_settings import get_write_file_compatibility_settings
|
||||
from deerflow.models.chat_stream_state import needs_qwen_flash_next_integrity
|
||||
from deerflow.reflection import resolve_class
|
||||
from deerflow.tracing import build_tracing_callbacks
|
||||
|
||||
@@ -31,6 +34,92 @@ def _vllm_disable_chat_template_kwargs(chat_template_kwargs: dict) -> dict:
|
||||
return disable_kwargs
|
||||
|
||||
|
||||
# Qwen3.8 chat templates only accept these three effort names.
|
||||
_QWEN_REASONING_EFFORT = {
|
||||
"minimal": "low",
|
||||
"low": "low",
|
||||
"medium": "medium",
|
||||
"high": "xhigh",
|
||||
"xhigh": "xhigh",
|
||||
}
|
||||
|
||||
_OPENAI_THINKING_OFF_EXTRA_BODY = {
|
||||
"enable_thinking": False,
|
||||
"chat_template_kwargs": {"enable_thinking": False},
|
||||
"thinking": {"type": "disabled"},
|
||||
}
|
||||
|
||||
|
||||
def _is_vllm_provider(model_use: str) -> bool:
|
||||
lowered = (model_use or "").lower()
|
||||
return "vllm_provider" in lowered or lowered.endswith(":vllmchatmodel")
|
||||
|
||||
|
||||
def _is_qwen38_model(model_config) -> bool:
|
||||
blob = f"{getattr(model_config, 'name', '')} {getattr(model_config, 'model', '')}".lower()
|
||||
return "qwen3.8" in blob
|
||||
|
||||
|
||||
def _should_control_openai_thinking(model_config) -> bool:
|
||||
"""Whether this OpenAI-compatible model needs explicit thinking on/off flags.
|
||||
|
||||
Qwen3.8 (and current vLLM reasoning parsers) default to thinking at ``xhigh``.
|
||||
If DeerFlow does not send ``enable_thinking: false``, the first visible answer
|
||||
token waits until that hidden reasoning finishes — third-party chat UIs that
|
||||
disable thinking or stream the reasoning field therefore feel much faster.
|
||||
"""
|
||||
if getattr(model_config, "supports_thinking", False):
|
||||
return True
|
||||
if _is_vllm_provider(getattr(model_config, "use", "")):
|
||||
return True
|
||||
return _is_qwen38_model(model_config)
|
||||
|
||||
|
||||
def _map_qwen_reasoning_effort(effort: object) -> str | None:
|
||||
if effort is None:
|
||||
return None
|
||||
return _QWEN_REASONING_EFFORT.get(str(effort).strip().lower())
|
||||
|
||||
|
||||
def _apply_openai_thinking_control(
|
||||
model_config,
|
||||
model_class,
|
||||
model_settings_from_config: dict,
|
||||
*,
|
||||
thinking_enabled: bool,
|
||||
) -> None:
|
||||
"""Inject vLLM/Qwen thinking flags when the UI toggle is on or off."""
|
||||
if "extra_body" not in getattr(model_class, "model_fields", {}):
|
||||
return
|
||||
qwen_vllm = _is_vllm_provider(getattr(model_config, "use", "")) or _is_qwen38_model(model_config)
|
||||
if thinking_enabled:
|
||||
if not qwen_vllm:
|
||||
return
|
||||
enable_body: dict = {
|
||||
"enable_thinking": True,
|
||||
"chat_template_kwargs": {"enable_thinking": True},
|
||||
}
|
||||
model_settings_from_config["extra_body"] = _deep_merge_dicts(
|
||||
model_settings_from_config.get("extra_body"),
|
||||
enable_body,
|
||||
)
|
||||
return
|
||||
if not _should_control_openai_thinking(model_config):
|
||||
return
|
||||
disable_body = (
|
||||
{
|
||||
"enable_thinking": False,
|
||||
"chat_template_kwargs": {"enable_thinking": False},
|
||||
}
|
||||
if qwen_vllm
|
||||
else _OPENAI_THINKING_OFF_EXTRA_BODY
|
||||
)
|
||||
model_settings_from_config["extra_body"] = _deep_merge_dicts(
|
||||
model_settings_from_config.get("extra_body"),
|
||||
disable_body,
|
||||
)
|
||||
|
||||
|
||||
def _enable_stream_usage_by_default(model_use_path: str, model_settings_from_config: dict) -> None:
|
||||
"""Enable stream usage for OpenAI-compatible models unless explicitly configured.
|
||||
|
||||
@@ -92,9 +181,50 @@ def create_chat_model(name: str | None = None, thinking_enabled: bool = False, *
|
||||
"when_thinking_enabled",
|
||||
"when_thinking_disabled",
|
||||
"thinking",
|
||||
"supports_vision",
|
||||
},
|
||||
)
|
||||
"supports_vision",
|
||||
"write_file_transport",
|
||||
},
|
||||
)
|
||||
configured_write_file_transport = model_config.write_file_transport
|
||||
runtime_write_file_compatibility = get_write_file_compatibility_settings()
|
||||
effective_write_file_transport = (
|
||||
WriteFileTransport.BUFFERED_GATEWAY
|
||||
if runtime_write_file_compatibility.enabled
|
||||
else configured_write_file_transport
|
||||
)
|
||||
if effective_write_file_transport in {
|
||||
WriteFileTransport.BUFFERED_GATEWAY,
|
||||
WriteFileTransport.ARTIFACT_RENDERER,
|
||||
WriteFileTransport.TEXT_BRIDGE,
|
||||
}:
|
||||
from deerflow.models.patched_openai import PatchedChatOpenAI
|
||||
|
||||
if not issubclass(model_class, PatchedChatOpenAI):
|
||||
raise ValueError(
|
||||
"write_file_transport=buffered_gateway/artifact_renderer/text_bridge is supported only by "
|
||||
"langchain_openai:ChatOpenAI or PatchedChatOpenAI models"
|
||||
)
|
||||
if model_config.use_responses_api:
|
||||
raise ValueError(
|
||||
"write_file_transport=buffered_gateway/artifact_renderer/text_bridge requires the "
|
||||
"OpenAI-compatible chat-completions API, not use_responses_api"
|
||||
)
|
||||
# The admin runtime switch overrides every compatible model on its next
|
||||
# construction. With the switch off, retain the legacy per-model value.
|
||||
model_settings_from_config["write_file_transport"] = effective_write_file_transport.value
|
||||
# ``max_input_tokens`` is a DeerFlow-side safety limit, not an OpenAI API
|
||||
# request parameter. ModelConfig intentionally allows provider-specific
|
||||
# extras, so without removing it here ChatOpenAI would forward the unknown
|
||||
# key to some compatible gateways and turn the protection itself into a
|
||||
# HTTP 400. Mirror it into LangChain's local-only ``profile`` metadata
|
||||
# when the selected model supports that field; the final context guard
|
||||
# reads the original model config and therefore also works for adapters
|
||||
# without profiles.
|
||||
configured_max_input_tokens = model_settings_from_config.pop("max_input_tokens", None)
|
||||
if configured_max_input_tokens is not None and "profile" in getattr(model_class, "model_fields", {}):
|
||||
profile = dict(model_settings_from_config.get("profile") or {})
|
||||
profile["max_input_tokens"] = configured_max_input_tokens
|
||||
model_settings_from_config["profile"] = profile
|
||||
# Compute effective when_thinking_enabled by merging in the `thinking` shortcut field.
|
||||
# The `thinking` shortcut is equivalent to setting when_thinking_enabled["thinking"].
|
||||
has_thinking_settings = (model_config.when_thinking_enabled is not None) or (model_config.thinking is not None)
|
||||
@@ -107,6 +237,13 @@ def create_chat_model(name: str | None = None, thinking_enabled: bool = False, *
|
||||
raise ValueError(f"Model {name} does not support thinking. Set `supports_thinking` to true in the `config.yaml` to enable thinking.") from None
|
||||
if effective_wte:
|
||||
model_settings_from_config.update(effective_wte)
|
||||
elif thinking_enabled and model_config.supports_thinking and model_config.use == "langchain_openai:ChatOpenAI":
|
||||
# vLLM/Qwen OpenAI-compatible endpoints commonly require this explicit
|
||||
# request flag before they emit delta.reasoning.
|
||||
model_settings_from_config["extra_body"] = _deep_merge_dicts(
|
||||
model_settings_from_config.get("extra_body"),
|
||||
{"enable_thinking": True},
|
||||
)
|
||||
if not thinking_enabled:
|
||||
if model_config.when_thinking_disabled is not None:
|
||||
# User-provided disable settings take full precedence
|
||||
@@ -127,9 +264,25 @@ def create_chat_model(name: str | None = None, thinking_enabled: bool = False, *
|
||||
elif has_thinking_settings and effective_wte.get("thinking", {}).get("type"):
|
||||
# Native langchain_anthropic: thinking is a direct constructor parameter
|
||||
model_settings_from_config["thinking"] = {"type": "disabled"}
|
||||
if not model_config.supports_reasoning_effort:
|
||||
kwargs.pop("reasoning_effort", None)
|
||||
# The Gateway validates the database-backed capability before constructing
|
||||
# the graph. Honor that server-validated runtime value without consulting a
|
||||
# legacy config.yaml flag. Qwen/vLLM accepts low/medium/xhigh; normalize the
|
||||
# old UI aliases defensively for internal callers that do not traverse the
|
||||
# public run boundary.
|
||||
requested_reasoning_effort = kwargs.get("reasoning_effort")
|
||||
if requested_reasoning_effort is not None:
|
||||
model_settings_from_config.pop("reasoning_effort", None)
|
||||
if _is_vllm_provider(model_config.use) or _is_qwen38_model(model_config):
|
||||
mapped_effort = _map_qwen_reasoning_effort(requested_reasoning_effort)
|
||||
if mapped_effort is not None:
|
||||
kwargs["reasoning_effort"] = mapped_effort
|
||||
|
||||
_apply_openai_thinking_control(
|
||||
model_config,
|
||||
model_class,
|
||||
model_settings_from_config,
|
||||
thinking_enabled=thinking_enabled,
|
||||
)
|
||||
|
||||
# Hard override: force-disable thinking regardless of the per-model config shape.
|
||||
# Injected only for OpenAI-compatible classes (those declaring `extra_body`);
|
||||
@@ -139,15 +292,10 @@ def create_chat_model(name: str | None = None, thinking_enabled: bool = False, *
|
||||
# (DeepSeek-V3.1+/Qwen3/GLM hybrid models on api.siliconflow.cn — the knob the
|
||||
# nested ``chat_template_kwargs`` shape does NOT reach), vLLM/Qwen native
|
||||
# ``chat_template_kwargs.enable_thinking``, and Zhipu GLM ``thinking.type``.
|
||||
# Only models that explicitly opt into thinking should receive vendor-specific
|
||||
# disable payloads. Generic OpenAI-compatible gateways (including LiteLLM)
|
||||
# may reject these otherwise-unknown fields instead of forwarding them.
|
||||
if (
|
||||
force_disable_thinking
|
||||
and not thinking_enabled
|
||||
and model_config.supports_thinking
|
||||
and "extra_body" in getattr(model_class, "model_fields", {})
|
||||
):
|
||||
# Only models that explicitly opt into thinking should receive vendor-specific
|
||||
# disable payloads. Generic OpenAI-compatible gateways (including LiteLLM)
|
||||
# may reject these otherwise-unknown fields instead of forwarding them.
|
||||
if force_disable_thinking and not thinking_enabled and model_config.supports_thinking and "extra_body" in getattr(model_class, "model_fields", {}):
|
||||
model_settings_from_config["extra_body"] = _deep_merge_dicts(
|
||||
model_settings_from_config.get("extra_body"),
|
||||
{"enable_thinking": False, "chat_template_kwargs": {"enable_thinking": False}, "thinking": {"type": "disabled"}},
|
||||
@@ -186,11 +334,35 @@ def create_chat_model(name: str | None = None, thinking_enabled: bool = False, *
|
||||
if "stream_usage" in getattr(model_class, "model_fields", {}):
|
||||
model_settings_from_config["stream_usage"] = True
|
||||
|
||||
model_instance = model_class(**kwargs, **model_settings_from_config)
|
||||
# The deployment may expose the affected model under a generic provider id.
|
||||
# Enable its local stream checks by either id or configured name, while
|
||||
# honoring an explicit override and keeping other models on their old path.
|
||||
if "strict_stream_integrity" in getattr(model_class, "model_fields", {}):
|
||||
if "strict_stream_integrity" in kwargs:
|
||||
model_settings_from_config["strict_stream_integrity"] = kwargs.pop("strict_stream_integrity")
|
||||
elif needs_qwen_flash_next_integrity(model_config.name, model_config.model):
|
||||
model_settings_from_config.setdefault("strict_stream_integrity", True)
|
||||
|
||||
callbacks = build_tracing_callbacks()
|
||||
if callbacks:
|
||||
existing_callbacks = model_instance.callbacks or []
|
||||
model_instance = model_class(**kwargs, **model_settings_from_config)
|
||||
|
||||
# Always emit a compact model lifecycle trace. In particular, the first
|
||||
# streamed callback proves that the upstream endpoint accepted the request
|
||||
# and started returning data, including reasoning-only chunks whose normal
|
||||
# content string is empty.
|
||||
from deerflow.models.timing_callback import ModelTimingCallback
|
||||
|
||||
existing_callbacks = model_instance.callbacks or []
|
||||
model_instance.callbacks = [
|
||||
*existing_callbacks,
|
||||
ModelTimingCallback(
|
||||
model_name=name,
|
||||
base_url=str(model_config.base_url) if model_config.base_url else None,
|
||||
),
|
||||
]
|
||||
|
||||
callbacks = build_tracing_callbacks()
|
||||
if callbacks:
|
||||
existing_callbacks = model_instance.callbacks or []
|
||||
model_instance.callbacks = [*existing_callbacks, *callbacks]
|
||||
logger.debug(f"Tracing attached to model '{name}' with providers={len(callbacks)}")
|
||||
return model_instance
|
||||
+29
-8
@@ -6,7 +6,6 @@ from typing import Any
|
||||
|
||||
from langchain_core.messages import BaseMessage, SystemMessage
|
||||
|
||||
|
||||
_SYSTEM_SEPARATOR = "\n\n---\n\n"
|
||||
|
||||
|
||||
@@ -35,7 +34,11 @@ def normalize_langchain_messages(messages: list[BaseMessage]) -> list[BaseMessag
|
||||
return messages
|
||||
|
||||
first_non_system_index = next((idx for idx, message in enumerate(messages) if not isinstance(message, SystemMessage)), len(messages))
|
||||
if len(system_messages) == first_non_system_index:
|
||||
# A number of strict chat templates accept *one* system message, and only
|
||||
# at index zero. Two adjacent leading system messages are therefore not a
|
||||
# valid no-op case: LangChain's agent prompt followed by DeerFlow's
|
||||
# transient compaction summary must still be folded into one message.
|
||||
if len(system_messages) == 1 and first_non_system_index == 1:
|
||||
return messages
|
||||
|
||||
merged = system_messages[0].model_copy(
|
||||
@@ -46,20 +49,38 @@ def normalize_langchain_messages(messages: list[BaseMessage]) -> list[BaseMessag
|
||||
return [merged, *(message for message in messages if not isinstance(message, SystemMessage))]
|
||||
|
||||
|
||||
def normalize_model_request_messages(
|
||||
system_message: SystemMessage | None,
|
||||
messages: list[BaseMessage],
|
||||
) -> tuple[SystemMessage | None, list[BaseMessage]]:
|
||||
"""Normalize LangChain's split ``ModelRequest`` message representation.
|
||||
|
||||
``ModelRequest.messages`` is documented as excluding the system prompt.
|
||||
Transient middleware (notably context compaction) can nevertheless place a
|
||||
``SystemMessage`` in that list. Fold every such message into the dedicated
|
||||
system slot so adapters without a DeerFlow-specific payload patch also
|
||||
receive exactly one leading system message.
|
||||
"""
|
||||
if system_message is None and not any(isinstance(message, SystemMessage) for message in messages):
|
||||
return None, messages
|
||||
|
||||
combined = [system_message, *messages] if system_message is not None else messages
|
||||
normalized = normalize_langchain_messages(combined)
|
||||
if normalized and isinstance(normalized[0], SystemMessage):
|
||||
return normalized[0], normalized[1:]
|
||||
return None, normalized
|
||||
|
||||
|
||||
def _normalize_payload_message_list(messages: list[Any]) -> list[Any]:
|
||||
system_messages = [message for message in messages if isinstance(message, dict) and message.get("role") == "system"]
|
||||
if not system_messages:
|
||||
return messages
|
||||
|
||||
first_non_system_index = next(
|
||||
(
|
||||
idx
|
||||
for idx, message in enumerate(messages)
|
||||
if not (isinstance(message, dict) and message.get("role") == "system")
|
||||
),
|
||||
(idx for idx, message in enumerate(messages) if not (isinstance(message, dict) and message.get("role") == "system")),
|
||||
len(messages),
|
||||
)
|
||||
if len(system_messages) == first_non_system_index:
|
||||
if len(system_messages) == 1 and first_non_system_index == 1:
|
||||
return messages
|
||||
|
||||
merged_system = dict(system_messages[0])
|
||||
|
||||
+490
@@ -0,0 +1,490 @@
|
||||
"""Keep one day of the exact text sent to a model and the text it sent back.
|
||||
|
||||
The admin page reads this store. Rows older than a day are dropped. Bodies are
|
||||
capped so a long tool argument cannot grow the file without limit. Recording
|
||||
never raises into the model call.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextvars
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
import uuid
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_RETENTION = timedelta(days=1)
|
||||
_STORE_NAME = "model-exchanges.jsonl"
|
||||
_MAX_RECORDS = 100
|
||||
_MAX_FIELD_CHARS = 200_000
|
||||
_PREVIEW_CHARS = 80
|
||||
_TRUNCATED = "\n…(超出保存长度,后面已截断)"
|
||||
|
||||
_LOCK = threading.Lock()
|
||||
_ACTIVE: contextvars.ContextVar[ExchangeCapture | None] = contextvars.ContextVar(
|
||||
"deerflow_model_exchange",
|
||||
default=None,
|
||||
)
|
||||
_store_override: Path | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExchangeCapture:
|
||||
"""One in-flight model call. Text stays here until the call finishes."""
|
||||
|
||||
model: str = ""
|
||||
enable_thinking: str = "unset"
|
||||
request_text: str = ""
|
||||
request_body: str = ""
|
||||
request_preview: str = ""
|
||||
content_parts: list[str] = field(default_factory=list)
|
||||
reasoning_by_field: dict[str, list[str]] = field(default_factory=dict)
|
||||
reasoning_fields: list[str] = field(default_factory=list)
|
||||
raw_chunks: list[str] = field(default_factory=list)
|
||||
owned: bool = False
|
||||
token: contextvars.Token | None = None
|
||||
saved: bool = False
|
||||
|
||||
|
||||
def use_exchange_store(path: Path | None) -> None:
|
||||
"""Point the store at a test file. ``None`` restores the runtime data dir."""
|
||||
global _store_override
|
||||
_store_override = path
|
||||
|
||||
|
||||
def open_exchange() -> ExchangeCapture:
|
||||
return ExchangeCapture()
|
||||
|
||||
|
||||
def bind_exchange(capture: ExchangeCapture) -> contextvars.Token:
|
||||
return _ACTIVE.set(capture)
|
||||
|
||||
|
||||
def unbind_exchange(token: contextvars.Token) -> None:
|
||||
_ACTIVE.reset(token)
|
||||
|
||||
|
||||
def activate_exchange() -> tuple[ExchangeCapture, contextvars.Token]:
|
||||
capture = open_exchange()
|
||||
return capture, bind_exchange(capture)
|
||||
|
||||
|
||||
def deactivate_exchange(capture: ExchangeCapture, token: contextvars.Token) -> None:
|
||||
try:
|
||||
finish_exchange(capture)
|
||||
finally:
|
||||
unbind_exchange(token)
|
||||
|
||||
|
||||
def remember_request(payload: Mapping[str, Any]) -> None:
|
||||
"""Copy the outbound messages once, after the payload is final."""
|
||||
try:
|
||||
capture = _ACTIVE.get()
|
||||
if capture is None:
|
||||
capture = ExchangeCapture(owned=True)
|
||||
capture.token = _ACTIVE.set(capture)
|
||||
elif capture.owned and capture.request_text and not capture.saved:
|
||||
_discard_owned(capture)
|
||||
capture = ExchangeCapture(owned=True)
|
||||
capture.token = _ACTIVE.set(capture)
|
||||
if capture.request_text:
|
||||
return
|
||||
messages = payload.get("messages")
|
||||
capture.model = str(payload.get("model") or "")
|
||||
capture.enable_thinking = _thinking_flag(payload)
|
||||
capture.request_preview = _preview(_last_user_text(messages))
|
||||
capture.request_text = _request_text(payload, messages)
|
||||
capture.request_body = _replay_request(payload)
|
||||
except Exception:
|
||||
logger.debug("model exchange request was not captured", exc_info=True)
|
||||
|
||||
|
||||
def remember_provider_chunk(chunk: Mapping[str, Any]) -> None:
|
||||
"""Append one provider chunk exactly as it arrived."""
|
||||
try:
|
||||
capture = _ACTIVE.get()
|
||||
if capture is None:
|
||||
return
|
||||
_store_raw(capture, chunk)
|
||||
for choice in _choices(chunk):
|
||||
delta = choice.get("delta") if isinstance(choice.get("delta"), Mapping) else {}
|
||||
message = choice.get("message") if isinstance(choice.get("message"), Mapping) else {}
|
||||
_append_channel(capture.content_parts, _text_field(delta, "content") or _text_field(message, "content"))
|
||||
_take_reasoning(capture, delta)
|
||||
_take_reasoning(capture, message)
|
||||
except Exception:
|
||||
logger.debug("model exchange chunk was not captured", exc_info=True)
|
||||
|
||||
|
||||
def remember_raw_response(payload: Mapping[str, Any]) -> None:
|
||||
"""Keep a non-streaming response body exactly as the provider sent it."""
|
||||
try:
|
||||
capture = _ACTIVE.get()
|
||||
if capture is None:
|
||||
return
|
||||
_store_raw(capture, payload, only_if_empty=True)
|
||||
except Exception:
|
||||
logger.debug("model exchange response was not captured", exc_info=True)
|
||||
|
||||
|
||||
def remember_final_message(message: Mapping[str, Any]) -> None:
|
||||
"""Record a non-streaming message when no chunks were captured."""
|
||||
try:
|
||||
capture = _ACTIVE.get()
|
||||
if capture is None:
|
||||
return
|
||||
_store_raw(capture, message, only_if_empty=True)
|
||||
if capture.content_parts or _has_reasoning(capture):
|
||||
return
|
||||
_append_channel(capture.content_parts, _text_field(message, "content"))
|
||||
_take_reasoning(capture, message)
|
||||
except Exception:
|
||||
logger.debug("model exchange message was not captured", exc_info=True)
|
||||
|
||||
|
||||
def finish_owned_exchange() -> None:
|
||||
capture = _ACTIVE.get()
|
||||
if capture is None or not capture.owned:
|
||||
return
|
||||
token = capture.token
|
||||
try:
|
||||
finish_exchange(capture)
|
||||
finally:
|
||||
if token is not None:
|
||||
_ACTIVE.reset(token)
|
||||
|
||||
|
||||
def finish_exchange(capture: ExchangeCapture) -> None:
|
||||
if capture.saved or not capture.request_text:
|
||||
capture.saved = True
|
||||
return
|
||||
capture.saved = True
|
||||
content = "".join(capture.content_parts)
|
||||
reasoning_by_field = {
|
||||
name: "".join(capture.reasoning_by_field.get(name, []))
|
||||
for name in capture.reasoning_fields
|
||||
if "".join(capture.reasoning_by_field.get(name, []))
|
||||
}
|
||||
reasoning = "".join(reasoning_by_field.values())
|
||||
reasoning_field = "、".join(reasoning_by_field) or "(没有单独的思考字段)"
|
||||
response_raw = _cap("\n\n".join(capture.raw_chunks))
|
||||
preview_source = content or reasoning
|
||||
record = {
|
||||
"id": uuid.uuid4().hex,
|
||||
"at": datetime.now(UTC).isoformat(timespec="seconds"),
|
||||
"model": capture.model,
|
||||
"user_id": _current_user_id(),
|
||||
"enable_thinking": capture.enable_thinking,
|
||||
"request_preview": capture.request_preview,
|
||||
"response_preview": _preview(preview_source),
|
||||
"request_text": _cap(capture.request_text),
|
||||
"request_body": capture.request_body,
|
||||
"response_text": response_raw,
|
||||
"response_raw": response_raw,
|
||||
"response_content": _cap(content),
|
||||
"response_reasoning": _cap(reasoning),
|
||||
"reasoning_field": reasoning_field,
|
||||
}
|
||||
_append_record(record)
|
||||
|
||||
|
||||
def query_model_exchanges() -> list[dict[str, Any]]:
|
||||
"""Newest first. Entries older than one day are dropped."""
|
||||
with _LOCK:
|
||||
records = _read_records()
|
||||
_write_records(records)
|
||||
return list(reversed(records))
|
||||
|
||||
|
||||
def _discard_owned(capture: ExchangeCapture) -> None:
|
||||
capture.saved = True
|
||||
token = capture.token
|
||||
if token is not None:
|
||||
_ACTIVE.reset(token)
|
||||
|
||||
|
||||
def _store_path() -> Path:
|
||||
if _store_override is not None:
|
||||
return _store_override
|
||||
from deerflow.config.paths import get_paths
|
||||
|
||||
return get_paths().base_dir / _STORE_NAME
|
||||
|
||||
|
||||
def _append_record(record: dict[str, Any]) -> None:
|
||||
try:
|
||||
with _LOCK:
|
||||
records = _read_records()
|
||||
records.append(record)
|
||||
if len(records) > _MAX_RECORDS:
|
||||
records = records[-_MAX_RECORDS:]
|
||||
_write_records(records)
|
||||
except Exception:
|
||||
logger.debug("model exchange was not saved", exc_info=True)
|
||||
|
||||
|
||||
def _read_records() -> list[dict[str, Any]]:
|
||||
path = _store_path()
|
||||
if not path.is_file():
|
||||
return []
|
||||
cutoff = datetime.now(UTC) - _RETENTION
|
||||
kept: list[dict[str, Any]] = []
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
record = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if isinstance(record, dict) and _record_time(record) >= cutoff:
|
||||
kept.append(record)
|
||||
if len(kept) > _MAX_RECORDS:
|
||||
kept = kept[-_MAX_RECORDS:]
|
||||
return kept
|
||||
|
||||
|
||||
def _write_records(records: list[dict[str, Any]]) -> None:
|
||||
path = _store_path()
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fd, tmp = tempfile.mkstemp(dir=path.parent, prefix=".model-exchanges.", suffix=".tmp")
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
||||
for record in records:
|
||||
handle.write(json.dumps(record, ensure_ascii=False) + "\n")
|
||||
os.replace(tmp, path)
|
||||
except Exception:
|
||||
try:
|
||||
os.unlink(tmp)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
def _record_time(record: Mapping[str, Any]) -> datetime:
|
||||
raw = str(record.get("at") or "")
|
||||
try:
|
||||
parsed = datetime.fromisoformat(raw)
|
||||
except ValueError:
|
||||
return datetime.min.replace(tzinfo=UTC)
|
||||
if parsed.tzinfo is None:
|
||||
return parsed.replace(tzinfo=UTC)
|
||||
return parsed.astimezone(UTC)
|
||||
|
||||
|
||||
def _current_user_id() -> str:
|
||||
try:
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
|
||||
return get_effective_user_id()
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
def _thinking_flag(payload: Mapping[str, Any]) -> str:
|
||||
extra_body = payload.get("extra_body")
|
||||
value = extra_body.get("enable_thinking", payload.get("enable_thinking")) if isinstance(extra_body, Mapping) else payload.get("enable_thinking")
|
||||
if value is True:
|
||||
return "true"
|
||||
if value is False:
|
||||
return "false"
|
||||
if value is None:
|
||||
return "unset"
|
||||
return "other"
|
||||
|
||||
|
||||
def _choices(chunk: Mapping[str, Any]) -> list[Mapping[str, Any]]:
|
||||
choices = chunk.get("choices")
|
||||
if not isinstance(choices, list) or not choices:
|
||||
nested = chunk.get("chunk")
|
||||
if isinstance(nested, Mapping):
|
||||
choices = nested.get("choices")
|
||||
if not isinstance(choices, list):
|
||||
return []
|
||||
return [choice for choice in choices if isinstance(choice, Mapping)]
|
||||
|
||||
|
||||
def _has_reasoning(capture: ExchangeCapture) -> bool:
|
||||
return any(parts for parts in capture.reasoning_by_field.values())
|
||||
|
||||
|
||||
def _take_reasoning(capture: ExchangeCapture, source: Mapping[str, Any]) -> None:
|
||||
"""Keep thinking text under the wire field that actually carried it."""
|
||||
for name in ("reasoning_content", "reasoning"):
|
||||
text = _text_field(source, name)
|
||||
if not text:
|
||||
continue
|
||||
if name not in capture.reasoning_fields:
|
||||
capture.reasoning_fields.append(name)
|
||||
_append_channel(capture.reasoning_by_field.setdefault(name, []), text)
|
||||
|
||||
|
||||
def _text_field(payload: Mapping[str, Any], key: str) -> str:
|
||||
value = payload.get(key)
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
return _content_to_text(value) if key == "content" else ""
|
||||
|
||||
|
||||
def _content_to_text(content: Any) -> str:
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
parts: list[str] = []
|
||||
for item in content:
|
||||
if isinstance(item, str):
|
||||
parts.append(item)
|
||||
elif isinstance(item, Mapping):
|
||||
text = item.get("text")
|
||||
if isinstance(text, str) and text:
|
||||
parts.append(text)
|
||||
elif item.get("type") == "image_url":
|
||||
parts.append("[image]")
|
||||
return "\n".join(parts)
|
||||
if content is None:
|
||||
return ""
|
||||
return str(content)
|
||||
|
||||
|
||||
def _last_user_text(messages: Any) -> str:
|
||||
if not isinstance(messages, list):
|
||||
return ""
|
||||
for message in reversed(messages):
|
||||
if isinstance(message, Mapping) and message.get("role") == "user":
|
||||
return _content_to_text(message.get("content"))
|
||||
return ""
|
||||
|
||||
|
||||
_SECRET_KEYS = frozenset({"api_key", "openai_api_key", "authorization", "token"})
|
||||
|
||||
|
||||
def _replay_request(payload: Mapping[str, Any]) -> str:
|
||||
"""JSON body the OpenAI client actually posts. ``extra_body`` is flattened."""
|
||||
ready = _json_ready(payload)
|
||||
if not isinstance(ready, dict):
|
||||
return "{}"
|
||||
extra = ready.pop("extra_body", None)
|
||||
if isinstance(extra, dict):
|
||||
ready.update(extra)
|
||||
return _fit_replay(ready)
|
||||
|
||||
|
||||
def _store_raw(capture: ExchangeCapture, payload: Mapping[str, Any], *, only_if_empty: bool = False) -> None:
|
||||
if only_if_empty and capture.raw_chunks:
|
||||
return
|
||||
if sum(len(part) for part in capture.raw_chunks) >= _MAX_FIELD_CHARS:
|
||||
return
|
||||
capture.raw_chunks.append(json.dumps(payload, ensure_ascii=False, indent=2, default=str))
|
||||
|
||||
|
||||
def _json_ready(value: Any) -> Any:
|
||||
if value is None or isinstance(value, (str, int, float, bool)):
|
||||
return value
|
||||
if isinstance(value, Mapping):
|
||||
return {
|
||||
str(key): _json_ready(item)
|
||||
for key, item in value.items()
|
||||
if item is not None and str(key) not in _SECRET_KEYS
|
||||
}
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [_json_ready(item) for item in value]
|
||||
return str(value)
|
||||
|
||||
|
||||
def _fit_replay(body: dict[str, Any]) -> str:
|
||||
encoded = json.dumps(body, ensure_ascii=False, indent=2)
|
||||
if len(encoded) <= _MAX_FIELD_CHARS:
|
||||
return encoded
|
||||
tools = body.get("tools")
|
||||
if isinstance(tools, list) and tools:
|
||||
names: list[str] = []
|
||||
for tool in tools:
|
||||
function = tool.get("function") if isinstance(tool, dict) else None
|
||||
name = function.get("name") if isinstance(function, dict) else None
|
||||
if isinstance(name, str) and name and name not in names:
|
||||
names.append(name)
|
||||
body["tools"] = [{"_omitted": "工具定义过长,这里只保留工具名", "names": names}]
|
||||
encoded = json.dumps(body, ensure_ascii=False, indent=2)
|
||||
if len(encoded) <= _MAX_FIELD_CHARS:
|
||||
return encoded
|
||||
messages = body.get("messages")
|
||||
if isinstance(messages, list):
|
||||
for _ in range(12):
|
||||
if len(encoded) <= _MAX_FIELD_CHARS:
|
||||
break
|
||||
longest: dict[str, Any] | None = None
|
||||
longest_len = 0
|
||||
for message in messages:
|
||||
if not isinstance(message, dict):
|
||||
continue
|
||||
content = message.get("content")
|
||||
if isinstance(content, str) and len(content) > longest_len:
|
||||
longest = message
|
||||
longest_len = len(content)
|
||||
if longest is None or longest_len < 200:
|
||||
break
|
||||
longest["content"] = longest["content"][: max(80, longest_len // 2)] + _TRUNCATED.strip()
|
||||
encoded = json.dumps(body, ensure_ascii=False, indent=2)
|
||||
if len(encoded) > _MAX_FIELD_CHARS:
|
||||
body["messages"] = [{"role": "user", "content": _TRUNCATED.strip()}]
|
||||
encoded = json.dumps(body, ensure_ascii=False, indent=2)
|
||||
return encoded
|
||||
|
||||
|
||||
def _request_text(payload: Mapping[str, Any], messages: Any) -> str:
|
||||
tool_names: list[str] = []
|
||||
tools = payload.get("tools")
|
||||
if isinstance(tools, list):
|
||||
for tool in tools:
|
||||
function = tool.get("function") if isinstance(tool, Mapping) else None
|
||||
name = function.get("name") if isinstance(function, Mapping) else None
|
||||
if isinstance(name, str) and name and name not in tool_names:
|
||||
tool_names.append(name)
|
||||
lines = [
|
||||
f"enable_thinking: {_thinking_flag(payload)}",
|
||||
f"tools: {', '.join(tool_names) if tool_names else '(无)'}",
|
||||
"",
|
||||
]
|
||||
if isinstance(messages, list):
|
||||
for message in messages:
|
||||
if not isinstance(message, Mapping):
|
||||
continue
|
||||
role = str(message.get("role") or "unknown")
|
||||
lines.append(f"[{role}]")
|
||||
lines.append(_content_to_text(message.get("content")))
|
||||
lines.append("")
|
||||
return "\n".join(lines).rstrip()
|
||||
|
||||
|
||||
def _append_channel(parts: list[str], incoming: str) -> None:
|
||||
if not incoming:
|
||||
return
|
||||
current = "".join(parts)
|
||||
if current and (incoming == current or incoming.startswith(current)):
|
||||
parts.clear()
|
||||
parts.append(incoming)
|
||||
return
|
||||
if current and current.endswith(incoming):
|
||||
return
|
||||
parts.append(incoming)
|
||||
|
||||
|
||||
def _preview(text: str) -> str:
|
||||
compact = " ".join(text.split())
|
||||
if len(compact) <= _PREVIEW_CHARS:
|
||||
return compact
|
||||
return compact[: _PREVIEW_CHARS - 1] + "…"
|
||||
|
||||
|
||||
def _cap(text: str) -> str:
|
||||
if len(text) <= _MAX_FIELD_CHARS:
|
||||
return text
|
||||
return text[: _MAX_FIELD_CHARS - len(_TRUNCATED)] + _TRUNCATED
|
||||
+4368
-67
File diff suppressed because it is too large.
Load diff
+140
@@ -0,0 +1,140 @@
|
||||
"""Normalize OpenAI-compatible thinking fields across provider versions.
|
||||
|
||||
Older vLLM / DeepSeek put thinking in ``reasoning_content``. Newer vLLM
|
||||
reasoning parsers put it in ``reasoning``. DeerFlow's UI and downstream code
|
||||
canonicalize on ``AIMessage.additional_kwargs.reasoning_content``.
|
||||
|
||||
Only fields that actually appeared on a response are echoed back on later
|
||||
turns, so models that still use ``reasoning_content`` (or send neither) are
|
||||
unchanged.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from langchain_core.messages import AIMessage, AIMessageChunk
|
||||
|
||||
|
||||
def reasoning_to_text(reasoning: Any) -> str:
|
||||
"""Best-effort extraction of readable reasoning text from provider payloads."""
|
||||
if isinstance(reasoning, str):
|
||||
return reasoning
|
||||
|
||||
if isinstance(reasoning, list):
|
||||
parts = [reasoning_to_text(item) for item in reasoning]
|
||||
return "".join(part for part in parts if part)
|
||||
|
||||
if isinstance(reasoning, dict):
|
||||
for key in ("text", "content", "reasoning"):
|
||||
value = reasoning.get(key)
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
if value is not None:
|
||||
text = reasoning_to_text(value)
|
||||
if text:
|
||||
return text
|
||||
try:
|
||||
return json.dumps(reasoning, ensure_ascii=False)
|
||||
except TypeError:
|
||||
return str(reasoning)
|
||||
|
||||
if reasoning is None:
|
||||
return ""
|
||||
try:
|
||||
return json.dumps(reasoning, ensure_ascii=False)
|
||||
except TypeError:
|
||||
return str(reasoning)
|
||||
|
||||
|
||||
def extract_reasoning_text(payload: dict | None) -> str | None:
|
||||
"""Read thinking text from ``reasoning_content`` or ``reasoning``.
|
||||
|
||||
Prefer ``reasoning_content`` when both are present so older DeepSeek /
|
||||
vLLM payloads keep their original string. Newer vLLM that only sends
|
||||
``reasoning`` (string or structured) is still captured.
|
||||
"""
|
||||
if not isinstance(payload, dict):
|
||||
return None
|
||||
for key in ("reasoning_content", "reasoning"):
|
||||
value = payload.get(key)
|
||||
if value is None:
|
||||
continue
|
||||
text = reasoning_to_text(value)
|
||||
if text:
|
||||
return text
|
||||
return None
|
||||
|
||||
|
||||
def _merge_streamed_value(existing: Any, incoming: Any) -> Any:
|
||||
if isinstance(existing, str) and isinstance(incoming, str):
|
||||
if incoming and incoming not in existing:
|
||||
return f"{existing}{incoming}"
|
||||
return existing or incoming
|
||||
return incoming if incoming is not None else existing
|
||||
|
||||
|
||||
def apply_reasoning_from_payload(
|
||||
message: AIMessage | AIMessageChunk,
|
||||
payload: dict | None,
|
||||
*,
|
||||
append: bool = True,
|
||||
) -> AIMessage | AIMessageChunk:
|
||||
"""Copy provider thinking fields onto ``additional_kwargs`` for UI + echo-back."""
|
||||
if not isinstance(payload, dict):
|
||||
return message
|
||||
|
||||
additional_kwargs = dict(message.additional_kwargs)
|
||||
changed = False
|
||||
|
||||
if "reasoning" in payload and payload.get("reasoning") is not None:
|
||||
incoming = payload.get("reasoning")
|
||||
existing = additional_kwargs.get("reasoning")
|
||||
merged = _merge_streamed_value(existing, incoming) if append else incoming
|
||||
if merged is not None and merged != existing:
|
||||
additional_kwargs["reasoning"] = merged
|
||||
changed = True
|
||||
|
||||
text = extract_reasoning_text(payload)
|
||||
if text:
|
||||
existing = additional_kwargs.get("reasoning_content")
|
||||
if append and isinstance(existing, str) and existing:
|
||||
if text not in existing:
|
||||
additional_kwargs["reasoning_content"] = f"{existing}{text}"
|
||||
changed = True
|
||||
elif text != existing:
|
||||
additional_kwargs["reasoning_content"] = text
|
||||
changed = True
|
||||
|
||||
if not changed:
|
||||
return message
|
||||
return message.model_copy(update={"additional_kwargs": additional_kwargs})
|
||||
|
||||
|
||||
def fill_ui_reasoning_content(message: AIMessage | AIMessageChunk) -> AIMessage | AIMessageChunk:
|
||||
"""If LangChain stored ``reasoning`` but not the UI field, copy the text across."""
|
||||
kwargs = message.additional_kwargs
|
||||
reasoning = kwargs.get("reasoning")
|
||||
if reasoning is None or kwargs.get("reasoning_content"):
|
||||
return message
|
||||
return apply_reasoning_from_payload(message, {"reasoning": reasoning}, append=False)
|
||||
|
||||
|
||||
def restore_reasoning_fields(payload_msg: dict, orig_msg: AIMessage) -> None:
|
||||
"""Re-inject the thinking field this assistant turn originally carried.
|
||||
|
||||
Newer vLLM stores thinking on ``reasoning`` and we also copy a UI-facing
|
||||
``reasoning_content`` string from it. Echo the wire field that actually
|
||||
arrived: ``reasoning`` wins when present so we do not send a synthesized
|
||||
``reasoning_content`` to a parser that no longer accepts it. Older
|
||||
DeepSeek / vLLM turns only have ``reasoning_content`` and stay on that
|
||||
field.
|
||||
"""
|
||||
reasoning = orig_msg.additional_kwargs.get("reasoning")
|
||||
if reasoning is not None:
|
||||
payload_msg["reasoning"] = reasoning
|
||||
return
|
||||
reasoning_content = orig_msg.additional_kwargs.get("reasoning_content")
|
||||
if reasoning_content is not None:
|
||||
payload_msg["reasoning_content"] = reasoning_content
|
||||
@@ -0,0 +1,260 @@
|
||||
"""Concise model connection and latency diagnostics for production logs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import atexit
|
||||
import logging
|
||||
import queue
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
from urllib.parse import urlsplit
|
||||
from uuid import UUID
|
||||
|
||||
from langchain_core.callbacks import BaseCallbackHandler
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class _TimingLogDispatcher:
|
||||
"""Keep slow log handlers off both the event loop and its tool executor.
|
||||
|
||||
Only already-sanitized, small lifecycle records enter this bounded queue.
|
||||
A daemon thread invokes the existing handlers; producers never wait for
|
||||
their I/O. Full queues drop diagnostics and report the count when the sink
|
||||
recovers, instead of retaining unbounded records or stalling generation.
|
||||
"""
|
||||
|
||||
def __init__(self, *, capacity: int = 4096) -> None:
|
||||
self._queue: queue.Queue[logging.LogRecord] = queue.Queue(maxsize=capacity)
|
||||
self._lock = threading.Lock()
|
||||
self._thread: threading.Thread | None = None
|
||||
self._closed = False
|
||||
self._dropped = 0
|
||||
|
||||
def emit(self, level: int, message: str, *args: Any) -> None:
|
||||
if not logger.isEnabledFor(level):
|
||||
return
|
||||
# Preserve the event's wall-clock timestamp even if delivery is delayed.
|
||||
# Formatting, filters and handler I/O run only on the consumer thread.
|
||||
record = logger.makeRecord(logger.name, level, __file__, 0, message, args, None)
|
||||
with self._lock:
|
||||
if self._closed:
|
||||
return
|
||||
try:
|
||||
self._queue.put_nowait(record)
|
||||
except queue.Full:
|
||||
self._dropped += 1
|
||||
return
|
||||
if self._thread is None:
|
||||
self._thread = threading.Thread(target=self._run, name="deerflow-model-timing-log", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def _run(self) -> None:
|
||||
while True:
|
||||
try:
|
||||
record = self._queue.get(timeout=0.1)
|
||||
except queue.Empty:
|
||||
with self._lock:
|
||||
if self._closed:
|
||||
return
|
||||
continue
|
||||
try:
|
||||
logger.handle(record)
|
||||
except Exception:
|
||||
# Diagnostic sinks must not fail a model call or kill the worker.
|
||||
pass
|
||||
finally:
|
||||
self._queue.task_done()
|
||||
with self._lock:
|
||||
dropped, self._dropped = self._dropped, 0
|
||||
if dropped:
|
||||
try:
|
||||
logger.warning("LLM_TIMING_LOG_DROPPED count=%d reason=queue_full", dropped)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def close(self, timeout: float = 0.5) -> None:
|
||||
"""Best-effort drain without making process exit wait on a log sink."""
|
||||
with self._lock:
|
||||
self._closed = True
|
||||
thread = self._thread
|
||||
if thread is not None and thread is not threading.current_thread():
|
||||
thread.join(timeout=timeout)
|
||||
|
||||
|
||||
_timing_logs = _TimingLogDispatcher()
|
||||
atexit.register(_timing_logs.close)
|
||||
|
||||
|
||||
def _safe_base_url(value: str | None) -> str:
|
||||
"""Log only the upstream origin, never credentials, query, or URL path."""
|
||||
if not value:
|
||||
return "-"
|
||||
try:
|
||||
parsed = urlsplit(value)
|
||||
except (TypeError, ValueError):
|
||||
return "-"
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
|
||||
return "-"
|
||||
host = parsed.hostname
|
||||
try:
|
||||
port = parsed.port
|
||||
except ValueError:
|
||||
return "-"
|
||||
return f"{parsed.scheme}://{host}:{port}" if port is not None else f"{parsed.scheme}://{host}"
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _CallTiming:
|
||||
started: float
|
||||
first_chunk: float | None = None
|
||||
|
||||
|
||||
class ModelTimingCallback(BaseCallbackHandler):
|
||||
"""Optionally record timing inline and deliver logs on a dedicated queue."""
|
||||
|
||||
# LangChain otherwise sends each synchronous callback, including every
|
||||
# token's no-op after the first, through the shared tool thread pool.
|
||||
# When enabled, inline work is limited to timestamps, short locked state
|
||||
# changes and a non-blocking enqueue, never synchronous handler I/O.
|
||||
run_inline = False
|
||||
|
||||
def __init__(self, *, model_name: str, base_url: str | None) -> None:
|
||||
from deerflow.config.runtime_performance_config import get_runtime_performance_config
|
||||
|
||||
self.run_inline = get_runtime_performance_config().model_timing_nonblocking_enabled
|
||||
self._model_name = model_name
|
||||
self._base_url = _safe_base_url(base_url)
|
||||
self._calls: dict[str, _CallTiming] = {}
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def _log(self, level: int, message: str, *args: Any) -> None:
|
||||
if self.run_inline:
|
||||
_timing_logs.emit(level, message, *args)
|
||||
else:
|
||||
# The default-off path retains LangChain's prior callback dispatch
|
||||
# and synchronous logging behavior for deployment compatibility.
|
||||
logger.log(level, message, *args)
|
||||
|
||||
def on_chat_model_start(
|
||||
self,
|
||||
serialized: dict[str, Any],
|
||||
messages: list[list[Any]],
|
||||
*,
|
||||
run_id: UUID,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
rid = str(run_id)
|
||||
now = time.monotonic()
|
||||
with self._lock:
|
||||
self._calls[rid] = _CallTiming(started=now)
|
||||
self._log(
|
||||
logging.INFO,
|
||||
"LLM_CONNECT_START model=%s base_url=%s run_id=%s message_batches=%d message_count=%d",
|
||||
self._model_name,
|
||||
self._base_url,
|
||||
rid,
|
||||
len(messages),
|
||||
sum(len(batch) for batch in messages),
|
||||
)
|
||||
|
||||
def on_llm_new_token(
|
||||
self,
|
||||
token: str,
|
||||
*,
|
||||
run_id: UUID,
|
||||
chunk: Any | None = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""Treat the first callback as connection success, including reasoning-only chunks."""
|
||||
rid = str(run_id)
|
||||
now = time.monotonic()
|
||||
with self._lock:
|
||||
timing = self._calls.get(rid)
|
||||
if timing is None or timing.first_chunk is not None:
|
||||
return
|
||||
timing.first_chunk = now
|
||||
first_chunk_ms = int((now - timing.started) * 1000)
|
||||
self._log(
|
||||
logging.INFO,
|
||||
"LLM_CONNECT_OK model=%s base_url=%s run_id=%s first_chunk_ms=%d",
|
||||
self._model_name,
|
||||
self._base_url,
|
||||
rid,
|
||||
first_chunk_ms,
|
||||
)
|
||||
|
||||
def on_llm_end(self, response: Any, *, run_id: UUID, **kwargs: Any) -> None:
|
||||
self._finish(run_id=run_id, error=None)
|
||||
|
||||
def on_llm_error(self, error: BaseException, *, run_id: UUID, **kwargs: Any) -> None:
|
||||
self._finish(run_id=run_id, error=error)
|
||||
|
||||
def _finish(self, *, run_id: UUID, error: BaseException | None) -> None:
|
||||
rid = str(run_id)
|
||||
now = time.monotonic()
|
||||
cancelled = isinstance(error, (asyncio.CancelledError, GeneratorExit))
|
||||
with self._lock:
|
||||
timing = self._calls.pop(rid, None)
|
||||
|
||||
if timing is None:
|
||||
if cancelled:
|
||||
self._log(
|
||||
logging.INFO,
|
||||
"LLM_REQUEST_CANCELLED model=%s base_url=%s run_id=%s status=cancelled error_type=%s",
|
||||
self._model_name,
|
||||
self._base_url,
|
||||
rid,
|
||||
type(error).__name__,
|
||||
)
|
||||
return
|
||||
self._log(
|
||||
logging.ERROR,
|
||||
"LLM_REQUEST_FAILED model=%s base_url=%s run_id=%s error_type=%s",
|
||||
self._model_name,
|
||||
self._base_url,
|
||||
rid,
|
||||
type(error).__name__ if error else "UnknownLifecycle",
|
||||
)
|
||||
return
|
||||
|
||||
total_ms = int((now - timing.started) * 1000)
|
||||
first_chunk_ms = int((timing.first_chunk - timing.started) * 1000) if timing.first_chunk else None
|
||||
if error is None:
|
||||
self._log(
|
||||
logging.INFO,
|
||||
"LLM_REQUEST_DONE model=%s base_url=%s run_id=%s status=success first_chunk_ms=%s total_ms=%d",
|
||||
self._model_name,
|
||||
self._base_url,
|
||||
rid,
|
||||
first_chunk_ms if first_chunk_ms is not None else "none",
|
||||
total_ms,
|
||||
)
|
||||
return
|
||||
|
||||
if cancelled:
|
||||
self._log(
|
||||
logging.INFO,
|
||||
"LLM_REQUEST_CANCELLED model=%s base_url=%s run_id=%s status=cancelled first_chunk_ms=%s total_ms=%d error_type=%s",
|
||||
self._model_name,
|
||||
self._base_url,
|
||||
rid,
|
||||
first_chunk_ms if first_chunk_ms is not None else "none",
|
||||
total_ms,
|
||||
type(error).__name__,
|
||||
)
|
||||
return
|
||||
|
||||
self._log(
|
||||
logging.ERROR,
|
||||
"LLM_REQUEST_FAILED model=%s base_url=%s run_id=%s status=error first_chunk_ms=%s total_ms=%d error_type=%s",
|
||||
self._model_name,
|
||||
self._base_url,
|
||||
rid,
|
||||
first_chunk_ms if first_chunk_ms is not None else "none",
|
||||
total_ms,
|
||||
type(error).__name__,
|
||||
)
|
||||
+31
-60
@@ -6,15 +6,16 @@ from assistant messages and streaming deltas. That breaks interleaved
|
||||
thinking/tool-call flows because vLLM expects the assistant's prior reasoning to
|
||||
be echoed back on subsequent turns.
|
||||
|
||||
This provider preserves ``reasoning`` on:
|
||||
- non-streaming responses
|
||||
- streaming deltas
|
||||
- multi-turn request payloads
|
||||
This provider preserves ``reasoning`` (newer vLLM) and
|
||||
``reasoning_content`` (older vLLM / DeepSeek-style parsers) on:
|
||||
- non-streaming responses
|
||||
- streaming deltas
|
||||
- multi-turn request payloads
|
||||
UI code still reads the canonical ``additional_kwargs.reasoning_content``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from typing import Any, cast
|
||||
|
||||
@@ -32,11 +33,12 @@ from langchain_core.messages import (
|
||||
)
|
||||
from langchain_core.messages.tool import tool_call_chunk
|
||||
from langchain_core.outputs import ChatGeneration, ChatGenerationChunk, ChatResult
|
||||
from langchain_openai import ChatOpenAI
|
||||
from langchain_openai.chat_models.base import _create_usage_metadata
|
||||
|
||||
from deerflow.models.message_ordering import normalize_chat_payload_messages
|
||||
|
||||
from langchain_openai import ChatOpenAI
|
||||
from langchain_openai.chat_models.base import _create_usage_metadata
|
||||
|
||||
from deerflow.models.message_ordering import normalize_chat_payload_messages
|
||||
from deerflow.models.reasoning_fields import apply_reasoning_from_payload, restore_reasoning_fields
|
||||
|
||||
|
||||
def _normalize_vllm_chat_template_kwargs(payload: dict[str, Any]) -> None:
|
||||
"""Map DeerFlow's legacy ``thinking`` toggle to vLLM/Qwen's ``enable_thinking``.
|
||||
@@ -66,31 +68,9 @@ def _normalize_vllm_chat_template_kwargs(payload: dict[str, Any]) -> None:
|
||||
|
||||
def _reasoning_to_text(reasoning: Any) -> str:
|
||||
"""Best-effort extraction of readable reasoning text from vLLM payloads."""
|
||||
if isinstance(reasoning, str):
|
||||
return reasoning
|
||||
from deerflow.models.reasoning_fields import reasoning_to_text
|
||||
|
||||
if isinstance(reasoning, list):
|
||||
parts = [_reasoning_to_text(item) for item in reasoning]
|
||||
return "".join(part for part in parts if part)
|
||||
|
||||
if isinstance(reasoning, dict):
|
||||
for key in ("text", "content", "reasoning"):
|
||||
value = reasoning.get(key)
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
if value is not None:
|
||||
text = _reasoning_to_text(value)
|
||||
if text:
|
||||
return text
|
||||
try:
|
||||
return json.dumps(reasoning, ensure_ascii=False)
|
||||
except TypeError:
|
||||
return str(reasoning)
|
||||
|
||||
try:
|
||||
return json.dumps(reasoning, ensure_ascii=False)
|
||||
except TypeError:
|
||||
return str(reasoning)
|
||||
return reasoning_to_text(reasoning)
|
||||
|
||||
|
||||
def _convert_delta_to_message_chunk_with_reasoning(_dict: Mapping[str, Any], default_class: type[BaseMessageChunk]) -> BaseMessageChunk:
|
||||
@@ -106,12 +86,8 @@ def _convert_delta_to_message_chunk_with_reasoning(_dict: Mapping[str, Any], def
|
||||
function_call["name"] = ""
|
||||
additional_kwargs["function_call"] = function_call
|
||||
|
||||
reasoning = _dict.get("reasoning")
|
||||
if reasoning is not None:
|
||||
additional_kwargs["reasoning"] = reasoning
|
||||
reasoning_text = _reasoning_to_text(reasoning)
|
||||
if reasoning_text:
|
||||
additional_kwargs["reasoning_content"] = reasoning_text
|
||||
seed = AIMessageChunk(content="", additional_kwargs=additional_kwargs)
|
||||
additional_kwargs = dict(apply_reasoning_from_payload(seed, dict(_dict)).additional_kwargs)
|
||||
|
||||
tool_call_chunks = []
|
||||
if raw_tool_calls := _dict.get("tool_calls"):
|
||||
@@ -151,11 +127,7 @@ def _convert_delta_to_message_chunk_with_reasoning(_dict: Mapping[str, Any], def
|
||||
|
||||
def _restore_reasoning_field(payload_msg: dict[str, Any], orig_msg: AIMessage) -> None:
|
||||
"""Re-inject vLLM reasoning onto outgoing assistant messages."""
|
||||
reasoning = orig_msg.additional_kwargs.get("reasoning")
|
||||
if reasoning is None:
|
||||
reasoning = orig_msg.additional_kwargs.get("reasoning_content")
|
||||
if reasoning is not None:
|
||||
payload_msg["reasoning"] = reasoning
|
||||
restore_reasoning_fields(payload_msg, orig_msg)
|
||||
|
||||
|
||||
class VllmChatModel(ChatOpenAI):
|
||||
@@ -184,14 +156,14 @@ class VllmChatModel(ChatOpenAI):
|
||||
for payload_msg, orig_msg in zip(payload_messages, original_messages):
|
||||
if payload_msg.get("role") == "assistant" and isinstance(orig_msg, AIMessage):
|
||||
_restore_reasoning_field(payload_msg, orig_msg)
|
||||
else:
|
||||
ai_messages = [message for message in original_messages if isinstance(message, AIMessage)]
|
||||
assistant_payloads = [message for message in payload_messages if message.get("role") == "assistant"]
|
||||
for payload_msg, ai_msg in zip(assistant_payloads, ai_messages):
|
||||
_restore_reasoning_field(payload_msg, ai_msg)
|
||||
|
||||
normalize_chat_payload_messages(payload)
|
||||
return payload
|
||||
else:
|
||||
ai_messages = [message for message in original_messages if isinstance(message, AIMessage)]
|
||||
assistant_payloads = [message for message in payload_messages if message.get("role") == "assistant"]
|
||||
for payload_msg, ai_msg in zip(assistant_payloads, ai_messages):
|
||||
_restore_reasoning_field(payload_msg, ai_msg)
|
||||
|
||||
normalize_chat_payload_messages(payload)
|
||||
return payload
|
||||
|
||||
def _create_chat_result(self, response: dict | openai.BaseModel, generation_info: dict | None = None) -> ChatResult:
|
||||
"""Preserve vLLM reasoning on non-streaming responses."""
|
||||
@@ -204,13 +176,12 @@ class VllmChatModel(ChatOpenAI):
|
||||
message = generation.message
|
||||
if not isinstance(message, AIMessage):
|
||||
continue
|
||||
reasoning = choice.get("message", {}).get("reasoning")
|
||||
if reasoning is None:
|
||||
continue
|
||||
message.additional_kwargs["reasoning"] = reasoning
|
||||
reasoning_text = _reasoning_to_text(reasoning)
|
||||
if reasoning_text:
|
||||
message.additional_kwargs["reasoning_content"] = reasoning_text
|
||||
choice_message = choice.get("message", {}) if isinstance(choice, dict) else {}
|
||||
generation.message = apply_reasoning_from_payload(
|
||||
message,
|
||||
choice_message if isinstance(choice_message, dict) else None,
|
||||
append=False,
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
Reference in new issue
Block a user