sync update from LAN repo (source master-nj @ 067eb808)
This commit is contained in:
1 parent
1244fb1f2f
commit
1d5dd8fc90
182 files changed
+18973
-769
No files matched your search
@@ -186,11 +186,26 @@ def create_chat_model(name: str | None = None, thinking_enabled: bool = False, *
|
||||
if "stream_usage" in getattr(model_class, "model_fields", {}):
|
||||
model_settings_from_config["stream_usage"] = True
|
||||
|
||||
model_instance = model_class(**kwargs, **model_settings_from_config)
|
||||
|
||||
callbacks = build_tracing_callbacks()
|
||||
if callbacks:
|
||||
existing_callbacks = model_instance.callbacks or []
|
||||
model_instance = model_class(**kwargs, **model_settings_from_config)
|
||||
|
||||
# Always emit a compact model lifecycle trace. In particular, the first
|
||||
# streamed callback proves that the upstream endpoint accepted the request
|
||||
# and started returning data, including reasoning-only chunks whose normal
|
||||
# content string is empty.
|
||||
from deerflow.models.timing_callback import ModelTimingCallback
|
||||
|
||||
existing_callbacks = model_instance.callbacks or []
|
||||
model_instance.callbacks = [
|
||||
*existing_callbacks,
|
||||
ModelTimingCallback(
|
||||
model_name=name,
|
||||
base_url=str(model_config.base_url) if model_config.base_url else None,
|
||||
),
|
||||
]
|
||||
|
||||
callbacks = build_tracing_callbacks()
|
||||
if callbacks:
|
||||
existing_callbacks = model_instance.callbacks or []
|
||||
model_instance.callbacks = [*existing_callbacks, *callbacks]
|
||||
logger.debug(f"Tracing attached to model '{name}' with providers={len(callbacks)}")
|
||||
return model_instance
|
||||
+29
-8
@@ -6,7 +6,6 @@ from typing import Any
|
||||
|
||||
from langchain_core.messages import BaseMessage, SystemMessage
|
||||
|
||||
|
||||
_SYSTEM_SEPARATOR = "\n\n---\n\n"
|
||||
|
||||
|
||||
@@ -35,7 +34,11 @@ def normalize_langchain_messages(messages: list[BaseMessage]) -> list[BaseMessag
|
||||
return messages
|
||||
|
||||
first_non_system_index = next((idx for idx, message in enumerate(messages) if not isinstance(message, SystemMessage)), len(messages))
|
||||
if len(system_messages) == first_non_system_index:
|
||||
# A number of strict chat templates accept *one* system message, and only
|
||||
# at index zero. Two adjacent leading system messages are therefore not a
|
||||
# valid no-op case: LangChain's agent prompt followed by DeerFlow's
|
||||
# transient compaction summary must still be folded into one message.
|
||||
if len(system_messages) == 1 and first_non_system_index == 1:
|
||||
return messages
|
||||
|
||||
merged = system_messages[0].model_copy(
|
||||
@@ -46,20 +49,38 @@ def normalize_langchain_messages(messages: list[BaseMessage]) -> list[BaseMessag
|
||||
return [merged, *(message for message in messages if not isinstance(message, SystemMessage))]
|
||||
|
||||
|
||||
def normalize_model_request_messages(
|
||||
system_message: SystemMessage | None,
|
||||
messages: list[BaseMessage],
|
||||
) -> tuple[SystemMessage | None, list[BaseMessage]]:
|
||||
"""Normalize LangChain's split ``ModelRequest`` message representation.
|
||||
|
||||
``ModelRequest.messages`` is documented as excluding the system prompt.
|
||||
Transient middleware (notably context compaction) can nevertheless place a
|
||||
``SystemMessage`` in that list. Fold every such message into the dedicated
|
||||
system slot so adapters without a DeerFlow-specific payload patch also
|
||||
receive exactly one leading system message.
|
||||
"""
|
||||
if system_message is None and not any(isinstance(message, SystemMessage) for message in messages):
|
||||
return None, messages
|
||||
|
||||
combined = [system_message, *messages] if system_message is not None else messages
|
||||
normalized = normalize_langchain_messages(combined)
|
||||
if normalized and isinstance(normalized[0], SystemMessage):
|
||||
return normalized[0], normalized[1:]
|
||||
return None, normalized
|
||||
|
||||
|
||||
def _normalize_payload_message_list(messages: list[Any]) -> list[Any]:
|
||||
system_messages = [message for message in messages if isinstance(message, dict) and message.get("role") == "system"]
|
||||
if not system_messages:
|
||||
return messages
|
||||
|
||||
first_non_system_index = next(
|
||||
(
|
||||
idx
|
||||
for idx, message in enumerate(messages)
|
||||
if not (isinstance(message, dict) and message.get("role") == "system")
|
||||
),
|
||||
(idx for idx, message in enumerate(messages) if not (isinstance(message, dict) and message.get("role") == "system")),
|
||||
len(messages),
|
||||
)
|
||||
if len(system_messages) == first_non_system_index:
|
||||
if len(system_messages) == 1 and first_non_system_index == 1:
|
||||
return messages
|
||||
|
||||
merged_system = dict(system_messages[0])
|
||||
|
||||
+140
-57
@@ -19,13 +19,15 @@ re-inject thinking metadata back into the outgoing payload for any assistant
|
||||
message that originally carried it:
|
||||
|
||||
- ``thought_signature`` for Gemini-compatible tool calls.
|
||||
- ``reasoning_content`` for providers (for example DeepSeek) that require it
|
||||
to be echoed in multi-turn requests when thinking mode is enabled.
|
||||
- ``reasoning_content`` and the newer ``reasoning`` spelling used by vLLM and
|
||||
other OpenAI-compatible providers. Both are normalized to
|
||||
``additional_kwargs.reasoning_content`` for DeerFlow's existing clients.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
from langchain_core.language_models import LanguageModelInput
|
||||
from langchain_core.messages import AIMessage, AIMessageChunk
|
||||
@@ -34,6 +36,65 @@ from langchain_openai import ChatOpenAI
|
||||
|
||||
from deerflow.models.message_ordering import normalize_chat_payload_messages
|
||||
|
||||
|
||||
class OpenAICompatibleUpstreamError(RuntimeError):
|
||||
"""An OpenAI-compatible gateway reported an error inside a HTTP 200 body.
|
||||
|
||||
Several compatible gateways do not use an HTTP error status for quota,
|
||||
concurrency, or inference failures. Instead they return ``choices=null``
|
||||
(or an empty streamed chunk) together with a vendor ``base_resp`` object.
|
||||
LangChain raises a generic ``TypeError`` for the non-streaming shape and
|
||||
treats the streaming shape as an empty successful response. Raising one
|
||||
stable exception here lets DeerFlow's model middleware retry both paths.
|
||||
"""
|
||||
|
||||
def __init__(self, *, code: Any, message: str | None = None) -> None:
|
||||
self.code = code
|
||||
detail = _bounded_error_message(message)
|
||||
suffix = f": {detail}" if detail else ""
|
||||
super().__init__(f"OpenAI-compatible upstream application error (code={code}){suffix}")
|
||||
|
||||
|
||||
def _bounded_error_message(value: Any, *, limit: int = 500) -> str:
|
||||
"""Return a log-safe, bounded one-line provider message."""
|
||||
if not isinstance(value, str):
|
||||
return ""
|
||||
normalized = " ".join(value.split())
|
||||
return normalized[:limit]
|
||||
|
||||
|
||||
def _application_error(payload: Mapping[str, Any]) -> tuple[Any, str | None] | None:
|
||||
"""Extract a non-success ``base_resp`` error from a compatible payload."""
|
||||
raw = payload.get("base_resp") or payload.get("baseResp")
|
||||
if not isinstance(raw, Mapping):
|
||||
return None
|
||||
|
||||
code = raw.get("status_code")
|
||||
if code is None:
|
||||
code = raw.get("statusCode")
|
||||
if code is None:
|
||||
code = raw.get("code")
|
||||
|
||||
# MiniMax-compatible responses use 0 for success. Be conservative for
|
||||
# gateways that put informational metadata in ``base_resp``.
|
||||
normalized_code = str(code).strip().lower() if code is not None else ""
|
||||
if normalized_code in {"", "0", "200", "ok", "success"}:
|
||||
return None
|
||||
|
||||
message = raw.get("status_msg")
|
||||
if message is None:
|
||||
message = raw.get("statusMsg")
|
||||
if message is None:
|
||||
message = raw.get("message")
|
||||
return code, message if isinstance(message, str) else None
|
||||
|
||||
|
||||
def _raise_for_application_error(payload: Mapping[str, Any]) -> None:
|
||||
error = _application_error(payload)
|
||||
if error is not None:
|
||||
code, message = error
|
||||
raise OpenAICompatibleUpstreamError(code=code, message=message)
|
||||
|
||||
|
||||
class PatchedChatOpenAI(ChatOpenAI):
|
||||
"""ChatOpenAI with ``thought_signature`` preservation for Gemini thinking via OpenAI gateway.
|
||||
@@ -100,18 +161,22 @@ class PatchedChatOpenAI(ChatOpenAI):
|
||||
normalize_chat_payload_messages(payload)
|
||||
return payload
|
||||
|
||||
def _convert_chunk_to_generation_chunk(
|
||||
self,
|
||||
chunk: dict,
|
||||
default_chunk_class: type,
|
||||
base_generation_info: dict | None,
|
||||
) -> ChatGenerationChunk | None:
|
||||
"""Preserve streamed ``reasoning_content`` in ``additional_kwargs``."""
|
||||
generation_chunk = super()._convert_chunk_to_generation_chunk(
|
||||
chunk,
|
||||
default_chunk_class,
|
||||
base_generation_info,
|
||||
)
|
||||
def _convert_chunk_to_generation_chunk(
|
||||
self,
|
||||
chunk: dict,
|
||||
default_chunk_class: type,
|
||||
base_generation_info: dict | None,
|
||||
) -> ChatGenerationChunk | None:
|
||||
"""Preserve streamed ``reasoning_content``/``reasoning`` metadata."""
|
||||
# Some compatible gateways return a vendor error event followed by a
|
||||
# normal-looking usage chunk, all under HTTP 200. Detect the first
|
||||
# event before LangChain converts it into an empty AIMessageChunk.
|
||||
_raise_for_application_error(chunk)
|
||||
generation_chunk = super()._convert_chunk_to_generation_chunk(
|
||||
chunk,
|
||||
default_chunk_class,
|
||||
base_generation_info,
|
||||
)
|
||||
if generation_chunk is None:
|
||||
return None
|
||||
|
||||
@@ -119,31 +184,37 @@ class PatchedChatOpenAI(ChatOpenAI):
|
||||
choice = choices[0] if choices else {}
|
||||
delta = choice.get("delta") if isinstance(choice, dict) else None
|
||||
reasoning_content = _extract_reasoning_content(delta if isinstance(delta, dict) else None)
|
||||
if reasoning_content and isinstance(generation_chunk.message, AIMessageChunk):
|
||||
generation_chunk.message = _append_reasoning_content(
|
||||
generation_chunk.message,
|
||||
reasoning_content,
|
||||
)
|
||||
if reasoning_content and isinstance(generation_chunk.message, AIMessageChunk):
|
||||
generation_chunk.message = _append_reasoning_content(
|
||||
generation_chunk.message,
|
||||
reasoning_content,
|
||||
reasoning=delta.get("reasoning") if isinstance(delta, dict) else None,
|
||||
)
|
||||
return generation_chunk
|
||||
|
||||
def _create_chat_result(
|
||||
self,
|
||||
response: dict | Any,
|
||||
generation_info: dict | None = None,
|
||||
) -> ChatResult:
|
||||
"""Preserve non-stream ``reasoning_content`` in ``additional_kwargs``."""
|
||||
result = super()._create_chat_result(response, generation_info)
|
||||
response_dict = response if isinstance(response, dict) else response.model_dump()
|
||||
choices = response_dict.get("choices", [])
|
||||
|
||||
generations: list[ChatGeneration] = []
|
||||
) -> ChatResult:
|
||||
"""Preserve non-stream ``reasoning_content``/``reasoning`` metadata."""
|
||||
response_dict = response if isinstance(response, dict) else response.model_dump()
|
||||
_raise_for_application_error(response_dict)
|
||||
result = super()._create_chat_result(response, generation_info)
|
||||
choices = response_dict.get("choices", [])
|
||||
|
||||
generations: list[ChatGeneration] = []
|
||||
for index, generation in enumerate(result.generations):
|
||||
message = generation.message
|
||||
choice = choices[index] if index < len(choices) else {}
|
||||
choice_message = choice.get("message", {}) if isinstance(choice, dict) else {}
|
||||
reasoning_content = _extract_reasoning_content(choice_message if isinstance(choice_message, dict) else None)
|
||||
if reasoning_content and isinstance(message, AIMessage):
|
||||
message = _append_reasoning_content(message, reasoning_content)
|
||||
if reasoning_content and isinstance(message, AIMessage):
|
||||
message = _append_reasoning_content(
|
||||
message,
|
||||
reasoning_content,
|
||||
reasoning=choice_message.get("reasoning") if isinstance(choice_message, dict) else None,
|
||||
)
|
||||
generation = ChatGeneration(
|
||||
message=message,
|
||||
generation_info=generation.generation_info,
|
||||
@@ -194,34 +265,46 @@ def _restore_tool_call_signatures(payload_msg: dict, orig_msg: AIMessage) -> Non
|
||||
payload_tc["thought_signature"] = sig
|
||||
|
||||
|
||||
def _restore_reasoning_content(payload_msg: dict, orig_msg: AIMessage) -> None:
|
||||
"""Re-inject ``reasoning_content`` onto assistant payload message.
|
||||
|
||||
Some thinking-enabled OpenAI-compatible providers require each historical
|
||||
assistant turn to include the original ``reasoning_content`` in subsequent
|
||||
requests. LangChain stores it in ``AIMessage.additional_kwargs`` but does
|
||||
not always serialize it back into ``messages``.
|
||||
"""
|
||||
reasoning_content = orig_msg.additional_kwargs.get("reasoning_content")
|
||||
if reasoning_content is not None:
|
||||
payload_msg["reasoning_content"] = reasoning_content
|
||||
|
||||
|
||||
def _extract_reasoning_content(payload: dict | None) -> str | None:
|
||||
"""Extract reasoning text from provider payload delta/message objects."""
|
||||
if not isinstance(payload, dict):
|
||||
return None
|
||||
reasoning_content = payload.get("reasoning_content")
|
||||
return reasoning_content if isinstance(reasoning_content, str) and reasoning_content else None
|
||||
|
||||
|
||||
def _append_reasoning_content(message: AIMessage | AIMessageChunk, reasoning_content: str) -> AIMessage | AIMessageChunk:
|
||||
"""Append streamed reasoning chunks, dedupe complete response reasoning."""
|
||||
additional_kwargs = dict(message.additional_kwargs)
|
||||
existing = additional_kwargs.get("reasoning_content")
|
||||
def _restore_reasoning_content(payload_msg: dict, orig_msg: AIMessage) -> None:
|
||||
"""Re-inject provider reasoning onto an assistant history message.
|
||||
|
||||
Some thinking-enabled OpenAI-compatible providers require each historical
|
||||
assistant turn to include its original reasoning in subsequent requests.
|
||||
Preserve the spelling returned by the provider when available; otherwise
|
||||
retain the established ``reasoning_content`` behavior.
|
||||
"""
|
||||
reasoning = orig_msg.additional_kwargs.get("reasoning")
|
||||
if reasoning is not None:
|
||||
payload_msg["reasoning"] = reasoning
|
||||
return
|
||||
|
||||
reasoning_content = orig_msg.additional_kwargs.get("reasoning_content")
|
||||
if reasoning_content is not None:
|
||||
payload_msg["reasoning_content"] = reasoning_content
|
||||
|
||||
|
||||
def _extract_reasoning_content(payload: dict | None) -> str | None:
|
||||
"""Extract either supported reasoning spelling from provider payloads."""
|
||||
if not isinstance(payload, dict):
|
||||
return None
|
||||
reasoning_content = payload.get("reasoning_content") or payload.get("reasoning")
|
||||
return reasoning_content if isinstance(reasoning_content, str) and reasoning_content else None
|
||||
|
||||
|
||||
def _append_reasoning_content(
|
||||
message: AIMessage | AIMessageChunk,
|
||||
reasoning_content: str,
|
||||
*,
|
||||
reasoning: Any = None,
|
||||
) -> AIMessage | AIMessageChunk:
|
||||
"""Normalize reasoning for clients while retaining its provider spelling."""
|
||||
additional_kwargs = dict(message.additional_kwargs)
|
||||
existing = additional_kwargs.get("reasoning_content")
|
||||
if isinstance(existing, str) and existing:
|
||||
if reasoning_content not in existing:
|
||||
additional_kwargs["reasoning_content"] = f"{existing}{reasoning_content}"
|
||||
else:
|
||||
additional_kwargs["reasoning_content"] = reasoning_content
|
||||
return message.model_copy(update={"additional_kwargs": additional_kwargs})
|
||||
else:
|
||||
additional_kwargs["reasoning_content"] = reasoning_content
|
||||
if reasoning is not None:
|
||||
additional_kwargs["reasoning"] = reasoning
|
||||
return message.model_copy(update={"additional_kwargs": additional_kwargs})
|
||||
@@ -0,0 +1,123 @@
|
||||
"""Concise model connection and latency diagnostics for production logs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from langchain_core.callbacks import BaseCallbackHandler
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _CallTiming:
|
||||
started: float
|
||||
first_chunk: float | None = None
|
||||
|
||||
|
||||
class ModelTimingCallback(BaseCallbackHandler):
|
||||
"""Log model-call start, first response chunk, completion, and failure."""
|
||||
|
||||
def __init__(self, *, model_name: str, base_url: str | None) -> None:
|
||||
self._model_name = model_name
|
||||
self._base_url = base_url or "-"
|
||||
self._calls: dict[str, _CallTiming] = {}
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def on_chat_model_start(
|
||||
self,
|
||||
serialized: dict[str, Any],
|
||||
messages: list[list[Any]],
|
||||
*,
|
||||
run_id: UUID,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
rid = str(run_id)
|
||||
now = time.monotonic()
|
||||
with self._lock:
|
||||
self._calls[rid] = _CallTiming(started=now)
|
||||
logger.info(
|
||||
"LLM_CONNECT_START model=%s base_url=%s run_id=%s message_batches=%d message_count=%d",
|
||||
self._model_name,
|
||||
self._base_url,
|
||||
rid,
|
||||
len(messages),
|
||||
sum(len(batch) for batch in messages),
|
||||
)
|
||||
|
||||
def on_llm_new_token(
|
||||
self,
|
||||
token: str,
|
||||
*,
|
||||
run_id: UUID,
|
||||
chunk: Any | None = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""Treat the first callback as connection success, including reasoning-only chunks."""
|
||||
rid = str(run_id)
|
||||
now = time.monotonic()
|
||||
with self._lock:
|
||||
timing = self._calls.get(rid)
|
||||
if timing is None or timing.first_chunk is not None:
|
||||
return
|
||||
timing.first_chunk = now
|
||||
first_chunk_ms = int((now - timing.started) * 1000)
|
||||
logger.info(
|
||||
"LLM_CONNECT_OK model=%s base_url=%s run_id=%s first_chunk_ms=%d",
|
||||
self._model_name,
|
||||
self._base_url,
|
||||
rid,
|
||||
first_chunk_ms,
|
||||
)
|
||||
|
||||
def on_llm_end(self, response: Any, *, run_id: UUID, **kwargs: Any) -> None:
|
||||
self._finish(run_id=run_id, error=None)
|
||||
|
||||
def on_llm_error(self, error: BaseException, *, run_id: UUID, **kwargs: Any) -> None:
|
||||
self._finish(run_id=run_id, error=error)
|
||||
|
||||
def _finish(self, *, run_id: UUID, error: BaseException | None) -> None:
|
||||
rid = str(run_id)
|
||||
now = time.monotonic()
|
||||
with self._lock:
|
||||
timing = self._calls.pop(rid, None)
|
||||
|
||||
if timing is None:
|
||||
logger.error(
|
||||
"LLM_REQUEST_FAILED model=%s base_url=%s run_id=%s error_type=%s error=%s",
|
||||
self._model_name,
|
||||
self._base_url,
|
||||
rid,
|
||||
type(error).__name__ if error else "UnknownLifecycle",
|
||||
error or "missing start event",
|
||||
)
|
||||
return
|
||||
|
||||
total_ms = int((now - timing.started) * 1000)
|
||||
first_chunk_ms = int((timing.first_chunk - timing.started) * 1000) if timing.first_chunk else None
|
||||
if error is None:
|
||||
logger.info(
|
||||
"LLM_REQUEST_DONE model=%s base_url=%s run_id=%s status=success first_chunk_ms=%s total_ms=%d",
|
||||
self._model_name,
|
||||
self._base_url,
|
||||
rid,
|
||||
first_chunk_ms if first_chunk_ms is not None else "none",
|
||||
total_ms,
|
||||
)
|
||||
return
|
||||
|
||||
logger.error(
|
||||
"LLM_REQUEST_FAILED model=%s base_url=%s run_id=%s status=error first_chunk_ms=%s total_ms=%d error_type=%s error=%s",
|
||||
self._model_name,
|
||||
self._base_url,
|
||||
rid,
|
||||
first_chunk_ms if first_chunk_ms is not None else "none",
|
||||
total_ms,
|
||||
type(error).__name__,
|
||||
error,
|
||||
)
|
||||
Reference in new issue
Block a user