sync update from LAN repo (source dhk-1 @ 3b1ec5b4)

This commit is contained in:
gozeros committed 2026-11-05 10:56:34 +08:00
1 parent da6521b6cb
commit 264e2200d6
995 files changed
+148105 -13399

No files matched your search

@@ -0,0 +1,106 @@
"""Invocation-local text and completion checks for compatible chat streams."""
from __future__ import annotations
from collections.abc import Mapping
from dataclasses import dataclass, field
from typing import Any
def needs_qwen_flash_next_integrity(*model_names: str | None) -> bool:
"""Match the affected model id or its local alias, without broad Qwen rules."""
return any(
name.strip().lower().replace("_", "-").rsplit("/", 1)[-1] == "qwen3.8-flash-next"
for name in model_names
if isinstance(name, str)
)
class IncompleteChatStreamError(RuntimeError):
"""A provider stream ended before a complete response could be committed.
Only a fixed reason is exposed. Provider text, URLs and error bodies must
never appear in an exception that the gateway may send to a browser.
"""
def __init__(self, reason: str) -> None:
self.reason = reason
super().__init__(f"Incomplete OpenAI-compatible chat stream: {reason}")
@dataclass
class ChatStreamState:
"""Track one request, never a model instance or a persisted conversation."""
observed_chat_frame: bool = False
terminal_received: bool = False
content: dict[int, str] = field(default_factory=dict)
reasoning: dict[int, str] = field(default_factory=dict)
def observe(self, choices: list[dict[str, Any]]) -> None:
self.observed_chat_frame = True
# LangChain consumes the first choice, so another choice's finish
# reason must not make an unfinished first choice look complete.
if choices and choices[0].get("finish_reason"):
self.terminal_received = True
def ensure_finished(self) -> None:
# Responses API and adapter-generated chunks do not pass through the
# chat-completions converter and have their own completion semantics.
if self.observed_chat_frame and not self.terminal_received:
raise IncompleteChatStreamError("missing_terminal_frame")
def normalize_text(
self,
*,
index: int,
delta: dict[str, Any],
original_delta: Mapping[str, Any] | None,
message: Mapping[str, Any] | None,
) -> None:
"""Convert explicit full-message snapshots to suffixes.
A real delta always wins and is appended verbatim, including repeated
words. Only text supplied through ``message`` is treated as a full
snapshot; prefix guessing on ordinary deltas would lose real tokens.
"""
for fields, accumulated in (
(("content",), self.content),
(("reasoning_content", "reasoning"), self.reasoning),
):
source = original_delta or {}
current = accumulated.get(index, "")
incoming = _first_text(source, fields)
if incoming is not None:
accumulated[index] = current + incoming[1]
continue
# Structured content and vendor reasoning objects are opaque.
# Preserve them instead of replacing them with a text snapshot.
if any(source.get(name) is not None and not isinstance(source[name], str) for name in fields):
continue
snapshot = _first_text(message or {}, fields)
if snapshot is None:
continue
name, text = snapshot
if text.startswith(current):
suffix = text[len(current):]
accumulated[index] = text
elif current.startswith(text):
# A duplicate/stale snapshot adds nothing to the live text.
suffix = ""
else:
# A delta cannot retract already-yielded bytes. Retry before
# committing contradictory text into the conversation.
raise IncompleteChatStreamError("inconsistent_message_snapshot")
for alias in fields:
delta.pop(alias, None)
delta[name] = suffix
def _first_text(source: Mapping[str, Any], fields: tuple[str, ...]) -> tuple[str, str] | None:
for name in fields:
value = source.get(name)
if isinstance(value, str) and value:
return name, value
return None
@@ -0,0 +1,460 @@
"""Recover DeepSeek DSML tool calls leaked through assistant text.
Some intranet OpenAI-compatible gateways render DeepSeek V4/V4.1 prompts
correctly but do not run the matching output parser. The model consequently
returns its native DSML envelope in ``message.content`` and leaves
``message.tool_calls`` empty. This module is a deliberately opt-in fallback
that converts only complete, schema-advertised calls back to OpenAI tool-call
chunks.
The parser is fail-closed:
* only tools advertised on the current provider request are accepted;
* names are matched exactly (misspellings are never corrected);
* an incomplete/malformed DSML envelope raises instead of executing a partial
call or exposing the protocol as ordinary assistant text;
* ordinary text is streamed without buffering except for the short prefix
needed to detect a marker split across provider chunks.
"""
from __future__ import annotations
import json
import re
import uuid
from collections.abc import Mapping
from dataclasses import dataclass, field
from typing import Any
from langchain_core.messages import AIMessageChunk
from langchain_core.outputs import ChatGenerationChunk
class DSMLToolCallProtocolError(RuntimeError):
"""A configured DSML provider returned an unsafe or incomplete call."""
@dataclass(frozen=True, slots=True)
class RecoveredToolCall:
name: str
arguments: dict[str, Any]
call_id: str
@dataclass(frozen=True, slots=True)
class ParsedDSMLContent:
content: str
calls: tuple[RecoveredToolCall, ...]
found: bool
_DSML_TOKEN = r"(?:|DSML||\|DSML\|)"
_OUTER_NAMES = r"(?:tool_calls|toolcalls|function_calls|calls|tool)"
_START_RE = re.compile(
rf"<\s*{_DSML_TOKEN}\s*{_OUTER_NAMES}\s*>",
re.IGNORECASE,
)
_END_RE = re.compile(
rf"</\s*{_DSML_TOKEN}\s*{_OUTER_NAMES}\s*>",
re.IGNORECASE,
)
_ANY_DSML_RE = re.compile(rf"<\s*/?\s*{_DSML_TOKEN}", re.IGNORECASE)
_INVOKE_RE = re.compile(
rf"<\s*{_DSML_TOKEN}\s*invoke\s+name\s*=\s*"
rf"(?P<quote>[\"'])(?P<name>[^\"']+)(?P=quote)\s*>"
rf"(?P<body>.*?)"
rf"</\s*{_DSML_TOKEN}\s*invoke\s*>",
re.IGNORECASE | re.DOTALL,
)
_PARAM_RE = re.compile(
rf"<\s*{_DSML_TOKEN}\s*parameter\s+name\s*=\s*"
rf"(?P<quote>[\"'])(?P<name>[^\"']+)(?P=quote)"
rf"(?:\s+string\s*=\s*(?P<squote>[\"'])(?P<string>true|false)(?P=squote))?\s*>"
rf"(?P<value>.*?)"
rf"</\s*{_DSML_TOKEN}\s*parameter\s*>",
re.IGNORECASE | re.DOTALL,
)
def _marker_candidates() -> tuple[str, ...]:
names = ("tool_calls", "toolcalls", "function_calls", "calls", "tool")
tokens = ("|DSML|", "|DSML|")
return tuple(
f"<{token}{space}{name}>"
for token in tokens
for space in ("", " ")
for name in names
)
_START_CANDIDATES = _marker_candidates()
def _longest_possible_start_suffix(text: str) -> int:
"""Length of the suffix that may be a split DSML opening marker."""
max_len = min(len(text), max(map(len, _START_CANDIDATES)))
for size in range(max_len, 0, -1):
suffix = text[-size:]
if any(candidate.startswith(suffix) for candidate in _START_CANDIDATES):
return size
return 0
def _parse_parameter_value(raw: str, string_flag: str | None) -> Any:
if (string_flag or "").lower() == "true":
return raw
try:
return json.loads(raw)
except (TypeError, ValueError) as exc:
if string_flag is None:
# DeepSeek occasionally omits the flag for an otherwise ordinary
# string parameter. Match the provider parser's safe literal
# fallback, while keeping explicit string=false strict.
return raw
raise DSMLToolCallProtocolError(
"DeepSeek DSML contained invalid JSON for a non-string parameter"
) from exc
def _parse_invoke_body(body: str) -> dict[str, Any]:
params: dict[str, Any] = {}
spans: list[tuple[int, int]] = []
for match in _PARAM_RE.finditer(body):
name = match.group("name").strip()
if not name or name in params:
raise DSMLToolCallProtocolError(
"DeepSeek DSML contained an empty or duplicate parameter name"
)
params[name] = _parse_parameter_value(
match.group("value"),
match.group("string"),
)
spans.append(match.span())
if spans:
remainder_parts: list[str] = []
cursor = 0
for start, end in spans:
remainder_parts.append(body[cursor:start])
cursor = end
remainder_parts.append(body[cursor:])
if "".join(remainder_parts).strip():
raise DSMLToolCallProtocolError(
"DeepSeek DSML invoke mixed parameter tags with unexpected text"
)
return params
stripped = body.strip()
if not stripped:
return {}
try:
direct = json.loads(stripped)
except (TypeError, ValueError) as exc:
raise DSMLToolCallProtocolError(
"DeepSeek DSML invoke contained neither parameters nor a JSON object"
) from exc
if not isinstance(direct, dict):
raise DSMLToolCallProtocolError(
"DeepSeek DSML direct arguments must be a JSON object"
)
return direct
def parse_dsml_content(
text: str,
*,
allowed_tools: set[str],
) -> ParsedDSMLContent:
"""Parse complete V4/V4.1 DSML blocks and remove them from content."""
if not text or not _ANY_DSML_RE.search(text):
return ParsedDSMLContent(content=text, calls=(), found=False)
calls: list[RecoveredToolCall] = []
visible_parts: list[str] = []
cursor = 0
found = False
while True:
start = _START_RE.search(text, cursor)
if start is None:
visible_parts.append(text[cursor:])
break
found = True
visible_parts.append(text[cursor : start.start()])
end = _END_RE.search(text, start.end())
if end is None:
raise DSMLToolCallProtocolError(
"DeepSeek DSML tool-call block was not closed"
)
inner = text[start.end() : end.start()]
invoke_spans: list[tuple[int, int]] = []
for invoke in _INVOKE_RE.finditer(inner):
name = invoke.group("name").strip()
if name not in allowed_tools:
raise DSMLToolCallProtocolError(
f"DeepSeek DSML requested an unadvertised tool: {name!r}"
)
calls.append(
RecoveredToolCall(
name=name,
arguments=_parse_invoke_body(invoke.group("body")),
call_id=f"call_deerflow_dsml_{uuid.uuid4().hex}",
)
)
invoke_spans.append(invoke.span())
if not invoke_spans:
raise DSMLToolCallProtocolError(
"DeepSeek DSML tool-call block contained no complete invoke"
)
remainder_parts: list[str] = []
inner_cursor = 0
for invoke_start, invoke_end in invoke_spans:
remainder_parts.append(inner[inner_cursor:invoke_start])
inner_cursor = invoke_end
remainder_parts.append(inner[inner_cursor:])
if "".join(remainder_parts).strip():
raise DSMLToolCallProtocolError(
"DeepSeek DSML tool-call block contained unexpected text"
)
cursor = end.end()
if not found:
# A DSML sentinel was present but did not match any supported complete
# envelope. Do not leak it as chat text or guess a repair.
raise DSMLToolCallProtocolError(
"DeepSeek DSML marker used an unsupported tool-call envelope"
)
return ParsedDSMLContent(
content="".join(visible_parts),
calls=tuple(calls),
found=True,
)
def _payload_tool_names(payload: Mapping[str, Any]) -> set[str]:
names: set[str] = set()
tools = payload.get("tools")
if not isinstance(tools, list):
return names
for tool in tools:
function = tool.get("function") if isinstance(tool, Mapping) else None
name = function.get("name") if isinstance(function, Mapping) else None
if isinstance(name, str) and name:
names.add(name)
return names
@dataclass(slots=True)
class DSMLToolCallRecovery:
"""Per-provider-invocation DSML compatibility state."""
enabled: bool
allowed_tools: set[str] = field(default_factory=set)
pending: str = ""
capture: str = ""
capture_template: ChatGenerationChunk | None = None
recovered: bool = False
def configure_from_payload(self, payload: Mapping[str, Any]) -> None:
if self.enabled:
self.allowed_tools = _payload_tool_names(payload)
def recover_nonstream_response(self, response: dict[str, Any]) -> None:
if not self.enabled or not self.allowed_tools:
return
choices = response.get("choices")
if not isinstance(choices, list):
return
for choice in choices:
if not isinstance(choice, dict):
continue
message = choice.get("message")
if not isinstance(message, dict) or message.get("tool_calls"):
continue
content = message.get("content")
if not isinstance(content, str):
continue
parsed = parse_dsml_content(content, allowed_tools=self.allowed_tools)
if not parsed.found:
continue
message["content"] = parsed.content
message["tool_calls"] = [
{
"id": call.call_id,
"type": "function",
"function": {
"name": call.name,
"arguments": json.dumps(
call.arguments,
ensure_ascii=False,
separators=(",", ":"),
),
},
}
for call in parsed.calls
]
choice["finish_reason"] = "tool_calls"
self.recovered = True
def feed(self, chunk: ChatGenerationChunk) -> list[ChatGenerationChunk]:
if not self.enabled or not self.allowed_tools:
return [chunk]
message = chunk.message
if not isinstance(message, AIMessageChunk):
return [chunk]
if message.tool_call_chunks or message.tool_calls:
if self.capture:
raise DSMLToolCallProtocolError(
"Provider mixed native tool calls with an incomplete DSML block"
)
return [chunk]
content = message.content
if not isinstance(content, str):
return [chunk]
if self.recovered:
if _ANY_DSML_RE.search(content):
raise DSMLToolCallProtocolError(
"Provider emitted another DSML block after a recovered tool call"
)
return [self._force_tool_finish(chunk)]
if self.capture:
self.capture += content
self.capture_template = chunk
return self._complete_capture_if_ready()
combined = self.pending + content
had_pending = bool(self.pending)
self.pending = ""
start = _START_RE.search(combined)
if start is not None:
before = combined[: start.start()]
self.capture = combined[start.start() :]
self.capture_template = chunk
output: list[ChatGenerationChunk] = []
if before or self._has_noncontent_signal(message):
output.append(self._replace_content(chunk, before))
output.extend(self._complete_capture_if_ready())
return output
hold = _longest_possible_start_suffix(combined)
if hold:
safe = combined[:-hold]
self.pending = combined[-hold:]
self.capture_template = chunk
if safe or self._has_noncontent_signal(message):
return [self._replace_content(chunk, safe)]
return []
return [self._replace_content(chunk, combined)] if had_pending else [chunk]
def finish(self) -> list[ChatGenerationChunk]:
if not self.enabled:
return []
if self.capture:
raise DSMLToolCallProtocolError(
"Provider stream ended inside a DeepSeek DSML tool-call block"
)
if not self.pending:
return []
pending = self.pending
self.pending = ""
if self.capture_template is None:
return []
return [self._replace_content(self.capture_template, pending)]
def _complete_capture_if_ready(self) -> list[ChatGenerationChunk]:
if _END_RE.search(self.capture) is None:
return []
parsed = parse_dsml_content(
self.capture,
allowed_tools=self.allowed_tools,
)
if not parsed.calls or self.capture_template is None:
raise DSMLToolCallProtocolError(
"DeepSeek DSML recovery produced no complete tool call"
)
template = self.capture_template
self.capture = ""
self.recovered = True
message = template.message
assert isinstance(message, AIMessageChunk)
tool_chunks = [
{
"name": call.name,
"args": json.dumps(
call.arguments,
ensure_ascii=False,
separators=(",", ":"),
),
"id": call.call_id,
"index": index,
"type": "tool_call_chunk",
}
for index, call in enumerate(parsed.calls)
]
response_metadata = dict(message.response_metadata)
if response_metadata.get("finish_reason"):
response_metadata["finish_reason"] = "tool_calls"
recovered_message = AIMessageChunk(
content=parsed.content,
additional_kwargs=dict(message.additional_kwargs),
response_metadata=response_metadata,
id=message.id,
name=message.name,
usage_metadata=message.usage_metadata,
tool_call_chunks=tool_chunks, # type: ignore[arg-type]
)
generation_info = dict(template.generation_info or {})
if generation_info.get("finish_reason"):
generation_info["finish_reason"] = "tool_calls"
return [
ChatGenerationChunk(
message=recovered_message,
generation_info=generation_info,
)
]
@staticmethod
def _replace_content(
chunk: ChatGenerationChunk,
content: str,
) -> ChatGenerationChunk:
return ChatGenerationChunk(
message=chunk.message.model_copy(update={"content": content}),
generation_info=chunk.generation_info,
)
@staticmethod
def _has_noncontent_signal(message: AIMessageChunk) -> bool:
return bool(
message.additional_kwargs
or message.response_metadata
or message.usage_metadata
)
@staticmethod
def _force_tool_finish(chunk: ChatGenerationChunk) -> ChatGenerationChunk:
message = chunk.message
if not isinstance(message, AIMessageChunk):
return chunk
response_metadata = dict(message.response_metadata)
if response_metadata.get("finish_reason"):
response_metadata["finish_reason"] = "tool_calls"
generation_info = dict(chunk.generation_info or {})
if generation_info.get("finish_reason"):
generation_info["finish_reason"] = "tool_calls"
return ChatGenerationChunk(
message=message.model_copy(update={"response_metadata": response_metadata}),
generation_info=generation_info or chunk.generation_info,
)
__all__ = [
"DSMLToolCallProtocolError",
"DSMLToolCallRecovery",
"ParsedDSMLContent",
"RecoveredToolCall",
"parse_dsml_content",
]
@@ -6,6 +6,7 @@ from deerflow.config import get_app_config
from deerflow.config.app_config import AppConfig
from deerflow.config.model_config import WriteFileTransport
from deerflow.config.system_settings import get_write_file_compatibility_settings
from deerflow.models.chat_stream_state import needs_qwen_flash_next_integrity
from deerflow.reflection import resolve_class
from deerflow.tracing import build_tracing_callbacks
@@ -86,7 +87,6 @@ def _apply_openai_thinking_control(
model_settings_from_config: dict,
*,
thinking_enabled: bool,
reasoning_effort: object,
) -> None:
"""Inject vLLM/Qwen thinking flags when the UI toggle is on or off."""
if "extra_body" not in getattr(model_class, "model_fields", {}):
@@ -99,9 +99,6 @@ def _apply_openai_thinking_control(
"enable_thinking": True,
"chat_template_kwargs": {"enable_thinking": True},
}
mapped_effort = _map_qwen_reasoning_effort(reasoning_effort)
if mapped_effort:
enable_body["chat_template_kwargs"]["reasoning_effort"] = mapped_effort
model_settings_from_config["extra_body"] = _deep_merge_dicts(
model_settings_from_config.get("extra_body"),
enable_body,
@@ -139,7 +136,16 @@ def _enable_stream_usage_by_default(model_use_path: str, model_settings_from_con
model_settings_from_config["stream_usage"] = True
def create_chat_model(name: str | None = None, thinking_enabled: bool = False, *, app_config: AppConfig | None = None, force_disable_thinking: bool = False, **kwargs) -> BaseChatModel:
def create_chat_model(
name: str | None = None,
thinking_enabled: bool = False,
*,
app_config: AppConfig | None = None,
force_disable_thinking: bool = False,
write_file_compatibility: bool | None = None,
prompt_suffix_enabled: bool = True,
**kwargs,
) -> BaseChatModel:
"""Create a chat model instance from the config.
Args:
@@ -172,8 +178,8 @@ def create_chat_model(name: str | None = None, thinking_enabled: bool = False, *
from deerflow.models.patched_openai import PatchedChatOpenAI
model_class = PatchedChatOpenAI
model_settings_from_config = model_config.model_dump(
exclude_none=True,
model_settings_from_config = model_config.model_dump(
exclude_none=True,
exclude={
"use",
"name",
@@ -186,15 +192,37 @@ def create_chat_model(name: str | None = None, thinking_enabled: bool = False, *
"thinking",
"supports_vision",
"write_file_transport",
"tool_call_transport",
},
)
configured_write_file_transport = model_config.write_file_transport
runtime_write_file_compatibility = get_write_file_compatibility_settings()
# ``None`` follows the global switch, so a roundtable run with its own
# switch off still uses compatibility when the global switch is on.
# ``True`` is the roundtable opt-in used when the global switch is off.
compatibility_enabled = (
get_write_file_compatibility_settings().enabled
if write_file_compatibility is None
else write_file_compatibility
)
effective_write_file_transport = (
WriteFileTransport.BUFFERED_GATEWAY
if runtime_write_file_compatibility.enabled
if compatibility_enabled
else configured_write_file_transport
)
if model_config.tool_call_transport.value != "native":
from deerflow.models.patched_openai import PatchedChatOpenAI
if not issubclass(model_class, PatchedChatOpenAI):
raise ValueError(
"tool_call_transport=deepseek_dsml is supported only by "
"langchain_openai:ChatOpenAI or PatchedChatOpenAI models"
)
if model_config.use_responses_api:
raise ValueError(
"tool_call_transport=deepseek_dsml requires the OpenAI-compatible "
"chat-completions API, not use_responses_api"
)
model_settings_from_config["tool_call_transport"] = model_config.tool_call_transport.value
if effective_write_file_transport in {
WriteFileTransport.BUFFERED_GATEWAY,
WriteFileTransport.ARTIFACT_RENDERER,
@@ -235,18 +263,18 @@ def create_chat_model(name: str | None = None, thinking_enabled: bool = False, *
if model_config.thinking is not None:
merged_thinking = {**(effective_wte.get("thinking") or {}), **model_config.thinking}
effective_wte = {**effective_wte, "thinking": merged_thinking}
if thinking_enabled and has_thinking_settings:
if not model_config.supports_thinking:
raise ValueError(f"Model {name} does not support thinking. Set `supports_thinking` to true in the `config.yaml` to enable thinking.") from None
if effective_wte:
model_settings_from_config.update(effective_wte)
elif thinking_enabled and model_config.supports_thinking and model_config.use == "langchain_openai:ChatOpenAI":
# vLLM/Qwen OpenAI-compatible endpoints commonly require this explicit
# request flag before they emit delta.reasoning.
model_settings_from_config["extra_body"] = _deep_merge_dicts(
model_settings_from_config.get("extra_body"),
{"enable_thinking": True},
)
if thinking_enabled and has_thinking_settings:
if not model_config.supports_thinking:
raise ValueError(f"Model {name} does not support thinking. Set `supports_thinking` to true in the `config.yaml` to enable thinking.") from None
if effective_wte:
model_settings_from_config.update(effective_wte)
elif thinking_enabled and model_config.supports_thinking and model_config.use == "langchain_openai:ChatOpenAI":
# vLLM/Qwen OpenAI-compatible endpoints commonly require this explicit
# request flag before they emit delta.reasoning.
model_settings_from_config["extra_body"] = _deep_merge_dicts(
model_settings_from_config.get("extra_body"),
{"enable_thinking": True},
)
if not thinking_enabled:
if model_config.when_thinking_disabled is not None:
# User-provided disable settings take full precedence
@@ -267,17 +295,24 @@ def create_chat_model(name: str | None = None, thinking_enabled: bool = False, *
elif has_thinking_settings and effective_wte.get("thinking", {}).get("type"):
# Native langchain_anthropic: thinking is a direct constructor parameter
model_settings_from_config["thinking"] = {"type": "disabled"}
# The Gateway validates the database-backed capability before constructing
# the graph. Honor that server-validated runtime value without consulting a
# legacy config.yaml flag. Qwen/vLLM accepts low/medium/xhigh; normalize the
# old UI aliases defensively for internal callers that do not traverse the
# public run boundary.
requested_reasoning_effort = kwargs.get("reasoning_effort")
if not model_config.supports_reasoning_effort:
kwargs.pop("reasoning_effort", None)
if requested_reasoning_effort is not None:
model_settings_from_config.pop("reasoning_effort", None)
if _is_vllm_provider(model_config.use) or _is_qwen38_model(model_config):
mapped_effort = _map_qwen_reasoning_effort(requested_reasoning_effort)
if mapped_effort is not None:
kwargs["reasoning_effort"] = mapped_effort
_apply_openai_thinking_control(
model_config,
model_class,
model_settings_from_config,
thinking_enabled=thinking_enabled,
reasoning_effort=requested_reasoning_effort,
)
# Hard override: force-disable thinking regardless of the per-model config shape.
@@ -291,12 +326,7 @@ def create_chat_model(name: str | None = None, thinking_enabled: bool = False, *
# Only models that explicitly opt into thinking should receive vendor-specific
# disable payloads. Generic OpenAI-compatible gateways (including LiteLLM)
# may reject these otherwise-unknown fields instead of forwarding them.
if (
force_disable_thinking
and not thinking_enabled
and model_config.supports_thinking
and "extra_body" in getattr(model_class, "model_fields", {})
):
if force_disable_thinking and not thinking_enabled and model_config.supports_thinking and "extra_body" in getattr(model_class, "model_fields", {}):
model_settings_from_config["extra_body"] = _deep_merge_dicts(
model_settings_from_config.get("extra_body"),
{"enable_thinking": False, "chat_template_kwargs": {"enable_thinking": False}, "thinking": {"type": "disabled"}},
@@ -335,7 +365,23 @@ def create_chat_model(name: str | None = None, thinking_enabled: bool = False, *
if "stream_usage" in getattr(model_class, "model_fields", {}):
model_settings_from_config["stream_usage"] = True
model_instance = model_class(**kwargs, **model_settings_from_config)
# The deployment may expose the affected model under a generic provider id.
# Enable its local stream checks by either id or configured name, while
# honoring an explicit override and keeping other models on their old path.
if "strict_stream_integrity" in getattr(model_class, "model_fields", {}):
if "strict_stream_integrity" in kwargs:
model_settings_from_config["strict_stream_integrity"] = kwargs.pop("strict_stream_integrity")
elif needs_qwen_flash_next_integrity(model_config.name, model_config.model):
model_settings_from_config.setdefault("strict_stream_integrity", True)
from deerflow.models.prompt_suffix import with_model_prompt_suffix
model_instance = with_model_prompt_suffix(model_class)(
**kwargs,
**model_settings_from_config,
deerflow_prompt_model_name=name,
deerflow_prompt_suffix_enabled=prompt_suffix_enabled,
)
# Always emit a compact model lifecycle trace. In particular, the first
# streamed callback proves that the upstream endpoint accepted the request
@@ -0,0 +1,490 @@
"""Keep one day of the exact text sent to a model and the text it sent back.
The admin page reads this store. Rows older than a day are dropped. Bodies are
capped so a long tool argument cannot grow the file without limit. Recording
never raises into the model call.
"""
from __future__ import annotations
import contextvars
import json
import logging
import os
import tempfile
import threading
import uuid
from collections.abc import Mapping
from dataclasses import dataclass, field
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
logger = logging.getLogger(__name__)
_RETENTION = timedelta(days=1)
_STORE_NAME = "model-exchanges.jsonl"
_MAX_RECORDS = 100
_MAX_FIELD_CHARS = 200_000
_PREVIEW_CHARS = 80
_TRUNCATED = "\n…(超出保存长度,后面已截断)"
_LOCK = threading.Lock()
_ACTIVE: contextvars.ContextVar[ExchangeCapture | None] = contextvars.ContextVar(
"deerflow_model_exchange",
default=None,
)
_store_override: Path | None = None
@dataclass
class ExchangeCapture:
"""One in-flight model call. Text stays here until the call finishes."""
model: str = ""
enable_thinking: str = "unset"
request_text: str = ""
request_body: str = ""
request_preview: str = ""
content_parts: list[str] = field(default_factory=list)
reasoning_by_field: dict[str, list[str]] = field(default_factory=dict)
reasoning_fields: list[str] = field(default_factory=list)
raw_chunks: list[str] = field(default_factory=list)
owned: bool = False
token: contextvars.Token | None = None
saved: bool = False
def use_exchange_store(path: Path | None) -> None:
"""Point the store at a test file. ``None`` restores the runtime data dir."""
global _store_override
_store_override = path
def open_exchange() -> ExchangeCapture:
return ExchangeCapture()
def bind_exchange(capture: ExchangeCapture) -> contextvars.Token:
return _ACTIVE.set(capture)
def unbind_exchange(token: contextvars.Token) -> None:
_ACTIVE.reset(token)
def activate_exchange() -> tuple[ExchangeCapture, contextvars.Token]:
capture = open_exchange()
return capture, bind_exchange(capture)
def deactivate_exchange(capture: ExchangeCapture, token: contextvars.Token) -> None:
try:
finish_exchange(capture)
finally:
unbind_exchange(token)
def remember_request(payload: Mapping[str, Any]) -> None:
"""Copy the outbound messages once, after the payload is final."""
try:
capture = _ACTIVE.get()
if capture is None:
capture = ExchangeCapture(owned=True)
capture.token = _ACTIVE.set(capture)
elif capture.owned and capture.request_text and not capture.saved:
_discard_owned(capture)
capture = ExchangeCapture(owned=True)
capture.token = _ACTIVE.set(capture)
if capture.request_text:
return
messages = payload.get("messages")
capture.model = str(payload.get("model") or "")
capture.enable_thinking = _thinking_flag(payload)
capture.request_preview = _preview(_last_user_text(messages))
capture.request_text = _request_text(payload, messages)
capture.request_body = _replay_request(payload)
except Exception:
logger.debug("model exchange request was not captured", exc_info=True)
def remember_provider_chunk(chunk: Mapping[str, Any]) -> None:
"""Append one provider chunk exactly as it arrived."""
try:
capture = _ACTIVE.get()
if capture is None:
return
_store_raw(capture, chunk)
for choice in _choices(chunk):
delta = choice.get("delta") if isinstance(choice.get("delta"), Mapping) else {}
message = choice.get("message") if isinstance(choice.get("message"), Mapping) else {}
_append_channel(capture.content_parts, _text_field(delta, "content") or _text_field(message, "content"))
_take_reasoning(capture, delta)
_take_reasoning(capture, message)
except Exception:
logger.debug("model exchange chunk was not captured", exc_info=True)
def remember_raw_response(payload: Mapping[str, Any]) -> None:
"""Keep a non-streaming response body exactly as the provider sent it."""
try:
capture = _ACTIVE.get()
if capture is None:
return
_store_raw(capture, payload, only_if_empty=True)
except Exception:
logger.debug("model exchange response was not captured", exc_info=True)
def remember_final_message(message: Mapping[str, Any]) -> None:
"""Record a non-streaming message when no chunks were captured."""
try:
capture = _ACTIVE.get()
if capture is None:
return
_store_raw(capture, message, only_if_empty=True)
if capture.content_parts or _has_reasoning(capture):
return
_append_channel(capture.content_parts, _text_field(message, "content"))
_take_reasoning(capture, message)
except Exception:
logger.debug("model exchange message was not captured", exc_info=True)
def finish_owned_exchange() -> None:
capture = _ACTIVE.get()
if capture is None or not capture.owned:
return
token = capture.token
try:
finish_exchange(capture)
finally:
if token is not None:
_ACTIVE.reset(token)
def finish_exchange(capture: ExchangeCapture) -> None:
if capture.saved or not capture.request_text:
capture.saved = True
return
capture.saved = True
content = "".join(capture.content_parts)
reasoning_by_field = {
name: "".join(capture.reasoning_by_field.get(name, []))
for name in capture.reasoning_fields
if "".join(capture.reasoning_by_field.get(name, []))
}
reasoning = "".join(reasoning_by_field.values())
reasoning_field = "、".join(reasoning_by_field) or "(没有单独的思考字段)"
response_raw = _cap("\n\n".join(capture.raw_chunks))
preview_source = content or reasoning
record = {
"id": uuid.uuid4().hex,
"at": datetime.now(UTC).isoformat(timespec="seconds"),
"model": capture.model,
"user_id": _current_user_id(),
"enable_thinking": capture.enable_thinking,
"request_preview": capture.request_preview,
"response_preview": _preview(preview_source),
"request_text": _cap(capture.request_text),
"request_body": capture.request_body,
"response_text": response_raw,
"response_raw": response_raw,
"response_content": _cap(content),
"response_reasoning": _cap(reasoning),
"reasoning_field": reasoning_field,
}
_append_record(record)
def query_model_exchanges() -> list[dict[str, Any]]:
"""Newest first. Entries older than one day are dropped."""
with _LOCK:
records = _read_records()
_write_records(records)
return list(reversed(records))
def _discard_owned(capture: ExchangeCapture) -> None:
capture.saved = True
token = capture.token
if token is not None:
_ACTIVE.reset(token)
def _store_path() -> Path:
if _store_override is not None:
return _store_override
from deerflow.config.paths import get_paths
return get_paths().base_dir / _STORE_NAME
def _append_record(record: dict[str, Any]) -> None:
try:
with _LOCK:
records = _read_records()
records.append(record)
if len(records) > _MAX_RECORDS:
records = records[-_MAX_RECORDS:]
_write_records(records)
except Exception:
logger.debug("model exchange was not saved", exc_info=True)
def _read_records() -> list[dict[str, Any]]:
path = _store_path()
if not path.is_file():
return []
cutoff = datetime.now(UTC) - _RETENTION
kept: list[dict[str, Any]] = []
for line in path.read_text(encoding="utf-8").splitlines():
if not line.strip():
continue
try:
record = json.loads(line)
except json.JSONDecodeError:
continue
if isinstance(record, dict) and _record_time(record) >= cutoff:
kept.append(record)
if len(kept) > _MAX_RECORDS:
kept = kept[-_MAX_RECORDS:]
return kept
def _write_records(records: list[dict[str, Any]]) -> None:
path = _store_path()
path.parent.mkdir(parents=True, exist_ok=True)
fd, tmp = tempfile.mkstemp(dir=path.parent, prefix=".model-exchanges.", suffix=".tmp")
try:
with os.fdopen(fd, "w", encoding="utf-8") as handle:
for record in records:
handle.write(json.dumps(record, ensure_ascii=False) + "\n")
os.replace(tmp, path)
except Exception:
try:
os.unlink(tmp)
except OSError:
pass
raise
def _record_time(record: Mapping[str, Any]) -> datetime:
raw = str(record.get("at") or "")
try:
parsed = datetime.fromisoformat(raw)
except ValueError:
return datetime.min.replace(tzinfo=UTC)
if parsed.tzinfo is None:
return parsed.replace(tzinfo=UTC)
return parsed.astimezone(UTC)
def _current_user_id() -> str:
try:
from deerflow.runtime.user_context import get_effective_user_id
return get_effective_user_id()
except Exception:
return ""
def _thinking_flag(payload: Mapping[str, Any]) -> str:
extra_body = payload.get("extra_body")
value = extra_body.get("enable_thinking", payload.get("enable_thinking")) if isinstance(extra_body, Mapping) else payload.get("enable_thinking")
if value is True:
return "true"
if value is False:
return "false"
if value is None:
return "unset"
return "other"
def _choices(chunk: Mapping[str, Any]) -> list[Mapping[str, Any]]:
choices = chunk.get("choices")
if not isinstance(choices, list) or not choices:
nested = chunk.get("chunk")
if isinstance(nested, Mapping):
choices = nested.get("choices")
if not isinstance(choices, list):
return []
return [choice for choice in choices if isinstance(choice, Mapping)]
def _has_reasoning(capture: ExchangeCapture) -> bool:
return any(parts for parts in capture.reasoning_by_field.values())
def _take_reasoning(capture: ExchangeCapture, source: Mapping[str, Any]) -> None:
"""Keep thinking text under the wire field that actually carried it."""
for name in ("reasoning_content", "reasoning"):
text = _text_field(source, name)
if not text:
continue
if name not in capture.reasoning_fields:
capture.reasoning_fields.append(name)
_append_channel(capture.reasoning_by_field.setdefault(name, []), text)
def _text_field(payload: Mapping[str, Any], key: str) -> str:
value = payload.get(key)
if isinstance(value, str):
return value
return _content_to_text(value) if key == "content" else ""
def _content_to_text(content: Any) -> str:
if isinstance(content, str):
return content
if isinstance(content, list):
parts: list[str] = []
for item in content:
if isinstance(item, str):
parts.append(item)
elif isinstance(item, Mapping):
text = item.get("text")
if isinstance(text, str) and text:
parts.append(text)
elif item.get("type") == "image_url":
parts.append("[image]")
return "\n".join(parts)
if content is None:
return ""
return str(content)
def _last_user_text(messages: Any) -> str:
if not isinstance(messages, list):
return ""
for message in reversed(messages):
if isinstance(message, Mapping) and message.get("role") == "user":
return _content_to_text(message.get("content"))
return ""
_SECRET_KEYS = frozenset({"api_key", "openai_api_key", "authorization", "token"})
def _replay_request(payload: Mapping[str, Any]) -> str:
"""JSON body the OpenAI client actually posts. ``extra_body`` is flattened."""
ready = _json_ready(payload)
if not isinstance(ready, dict):
return "{}"
extra = ready.pop("extra_body", None)
if isinstance(extra, dict):
ready.update(extra)
return _fit_replay(ready)
def _store_raw(capture: ExchangeCapture, payload: Mapping[str, Any], *, only_if_empty: bool = False) -> None:
if only_if_empty and capture.raw_chunks:
return
if sum(len(part) for part in capture.raw_chunks) >= _MAX_FIELD_CHARS:
return
capture.raw_chunks.append(json.dumps(payload, ensure_ascii=False, indent=2, default=str))
def _json_ready(value: Any) -> Any:
if value is None or isinstance(value, (str, int, float, bool)):
return value
if isinstance(value, Mapping):
return {
str(key): _json_ready(item)
for key, item in value.items()
if item is not None and str(key) not in _SECRET_KEYS
}
if isinstance(value, (list, tuple)):
return [_json_ready(item) for item in value]
return str(value)
def _fit_replay(body: dict[str, Any]) -> str:
encoded = json.dumps(body, ensure_ascii=False, indent=2)
if len(encoded) <= _MAX_FIELD_CHARS:
return encoded
tools = body.get("tools")
if isinstance(tools, list) and tools:
names: list[str] = []
for tool in tools:
function = tool.get("function") if isinstance(tool, dict) else None
name = function.get("name") if isinstance(function, dict) else None
if isinstance(name, str) and name and name not in names:
names.append(name)
body["tools"] = [{"_omitted": "工具定义过长,这里只保留工具名", "names": names}]
encoded = json.dumps(body, ensure_ascii=False, indent=2)
if len(encoded) <= _MAX_FIELD_CHARS:
return encoded
messages = body.get("messages")
if isinstance(messages, list):
for _ in range(12):
if len(encoded) <= _MAX_FIELD_CHARS:
break
longest: dict[str, Any] | None = None
longest_len = 0
for message in messages:
if not isinstance(message, dict):
continue
content = message.get("content")
if isinstance(content, str) and len(content) > longest_len:
longest = message
longest_len = len(content)
if longest is None or longest_len < 200:
break
longest["content"] = longest["content"][: max(80, longest_len // 2)] + _TRUNCATED.strip()
encoded = json.dumps(body, ensure_ascii=False, indent=2)
if len(encoded) > _MAX_FIELD_CHARS:
body["messages"] = [{"role": "user", "content": _TRUNCATED.strip()}]
encoded = json.dumps(body, ensure_ascii=False, indent=2)
return encoded
def _request_text(payload: Mapping[str, Any], messages: Any) -> str:
tool_names: list[str] = []
tools = payload.get("tools")
if isinstance(tools, list):
for tool in tools:
function = tool.get("function") if isinstance(tool, Mapping) else None
name = function.get("name") if isinstance(function, Mapping) else None
if isinstance(name, str) and name and name not in tool_names:
tool_names.append(name)
lines = [
f"enable_thinking: {_thinking_flag(payload)}",
f"tools: {', '.join(tool_names) if tool_names else '(无)'}",
"",
]
if isinstance(messages, list):
for message in messages:
if not isinstance(message, Mapping):
continue
role = str(message.get("role") or "unknown")
lines.append(f"[{role}]")
lines.append(_content_to_text(message.get("content")))
lines.append("")
return "\n".join(lines).rstrip()
def _append_channel(parts: list[str], incoming: str) -> None:
if not incoming:
return
current = "".join(parts)
if current and (incoming == current or incoming.startswith(current)):
parts.clear()
parts.append(incoming)
return
if current and current.endswith(incoming):
return
parts.append(incoming)
def _preview(text: str) -> str:
compact = " ".join(text.split())
if len(compact) <= _PREVIEW_CHARS:
return compact
return compact[: _PREVIEW_CHARS - 1] + "…"
def _cap(text: str) -> str:
if len(text) <= _MAX_FIELD_CHARS:
return text
return text[: _MAX_FIELD_CHARS - len(_TRUNCATED)] + _TRUNCATED
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,121 @@
"""Apply deployment instructions at the model boundary without editing chat state."""
from functools import lru_cache
from langchain_core.messages import BaseMessage, SystemMessage
from langchain_core.prompt_values import ChatPromptValue
from pydantic import Field
from deerflow.config.system_settings import load_system_settings
from deerflow.models.message_ordering import normalize_langchain_messages
def append_model_prompt_suffix(
messages: list[BaseMessage],
model_name: str,
*,
enabled: bool = True,
) -> list[BaseMessage]:
if not enabled or not model_name:
return messages
setting = load_system_settings().model_prompt_suffixes.get(model_name)
if setting is None or not setting.enabled or not setting.prompt.strip():
return messages
suffix = setting.prompt.strip()
normalized = normalize_langchain_messages(messages)
if not normalized or not isinstance(normalized[0], SystemMessage):
return [SystemMessage(content=suffix), *normalized]
system = normalized[0]
content = system.content
# invoke converts input before generate; fallback streaming can also pass
# through both entry points. Do not append the same instructions twice.
if isinstance(content, str):
if content == suffix or content.endswith("\n\n" + suffix):
return normalized
content = content + "\n\n" + suffix
else:
block = {"type": "text", "text": "\n\n" + suffix}
if content and content[-1] == block:
return normalized
content = [*content, block]
return [system.model_copy(update={"content": content}), *normalized[1:]]
def append_payload_prompt_suffix(
payload: dict,
model_name: str,
*,
enabled: bool = True,
) -> None:
"""Append after OpenAI transport adapters have added their own instructions."""
key = "messages" if "messages" in payload else "input"
if not model_name or not isinstance(payload.get(key), list):
return
messages = payload[key]
system_index = next((i for i in range(len(messages) - 1, -1, -1) if messages[i].get("role") in {"system", "developer"}), None)
system = messages[system_index] if system_index is not None else None
original = [SystemMessage(content=system["content"])] if system else []
updated = append_model_prompt_suffix(original, model_name, enabled=enabled)
if updated is original:
return
content = updated[0].content
if key == "input" and isinstance(content, list):
content = [{**block, "type": "input_text"} if isinstance(block, dict) and block.get("type") == "text" else block for block in content]
if system is not None:
payload[key] = [*messages]
payload[key][system_index] = {**system, "content": content}
else:
payload[key] = [{"role": "system", "content": content}, *messages]
@lru_cache(maxsize=64)
def with_model_prompt_suffix(model_class: type) -> type:
"""Keep the provider's tool binding/structured output and native streaming.
Read settings per request, including for already-created model instances.
The local routing name is excluded from serialization/provider parameters.
"""
from deerflow.models.patched_openai import PatchedChatOpenAI
uses_payload_suffix = issubclass(model_class, PatchedChatOpenAI)
class ModelWithPromptSuffix(model_class):
deerflow_prompt_model_name: str = Field(default="", exclude=True)
deerflow_prompt_suffix_enabled: bool = Field(default=True, exclude=True)
def _convert_input(self, model_input):
prompt = super()._convert_input(model_input)
if uses_payload_suffix:
return prompt
return ChatPromptValue(messages=append_model_prompt_suffix(
prompt.to_messages(),
self.deerflow_prompt_model_name,
enabled=self.deerflow_prompt_suffix_enabled,
))
def generate(self, messages, *args, **kwargs):
if uses_payload_suffix:
return super().generate(messages, *args, **kwargs)
return super().generate([
append_model_prompt_suffix(
batch,
self.deerflow_prompt_model_name,
enabled=self.deerflow_prompt_suffix_enabled,
)
for batch in messages
], *args, **kwargs)
async def agenerate(self, messages, *args, **kwargs):
if uses_payload_suffix:
return await super().agenerate(messages, *args, **kwargs)
return await super().agenerate([
append_model_prompt_suffix(
batch,
self.deerflow_prompt_model_name,
enabled=self.deerflow_prompt_suffix_enabled,
)
for batch in messages
], *args, **kwargs)
return ModelWithPromptSuffix