sync update from LAN repo (source ji @ 85078a31)
This commit is contained in:
1 parent
5d6627af50
commit
e602d88695
864 files changed
+99150
-6346
No files matched your search
@@ -29,14 +29,14 @@ from app.gateway.authz import require_permission
|
||||
from app.gateway.deps import get_checkpointer
|
||||
from app.gateway.llmwiki_deposit import delete_conversation_deposit_documents
|
||||
from app.gateway.utils import sanitize_log_param
|
||||
from deerflow.config.extensions_config import SkillDisplayConfig
|
||||
from deerflow.config.paths import Paths, get_paths
|
||||
from deerflow.persistence.engine import get_engine, get_session_factory
|
||||
from deerflow.persistence.skills.model import SkillDisplayAdapterRow
|
||||
from deerflow.agents.roundtable_orchestrator.delivery import (
|
||||
classify_seat_delivery,
|
||||
visible_seat_text,
|
||||
)
|
||||
from deerflow.config.extensions_config import SkillDisplayConfig
|
||||
from deerflow.config.paths import Paths, get_paths
|
||||
from deerflow.persistence.engine import get_engine, get_session_factory
|
||||
from deerflow.persistence.skills.model import SkillDisplayAdapterRow
|
||||
from deerflow.runtime import serialize_channel_values
|
||||
from deerflow.runtime.references import (
|
||||
adapt_reference_batches_with_displays,
|
||||
@@ -272,6 +272,7 @@ class ReferenceBatchResponse(BaseModel):
|
||||
|
||||
class ReferenceBatchesResponse(BaseModel):
|
||||
batches: list[ReferenceBatchResponse] = Field(default_factory=list)
|
||||
retrieval_traces: list[dict[str, Any]] = Field(default_factory=list)
|
||||
|
||||
|
||||
class ReferenceDebugLogRequest(BaseModel):
|
||||
@@ -280,6 +281,150 @@ class ReferenceDebugLogRequest(BaseModel):
|
||||
payload: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
def _trace_message_extra(message: Any) -> dict[str, Any]:
|
||||
if isinstance(message, dict):
|
||||
value = message.get("additional_kwargs") or message.get("additionalKwargs")
|
||||
else:
|
||||
value = getattr(message, "additional_kwargs", None)
|
||||
return value if isinstance(value, dict) else {}
|
||||
|
||||
|
||||
def _trace_message_value(message: Any, key: str) -> Any:
|
||||
if isinstance(message, dict):
|
||||
return message.get(key)
|
||||
return getattr(message, key, None)
|
||||
|
||||
|
||||
def _tool_trace_from_message(message: Any) -> dict[str, Any] | None:
|
||||
"""Build a bounded trace for a model-driven ``llmwiki_search`` call."""
|
||||
|
||||
message_type = str(_trace_message_value(message, "type") or "").lower()
|
||||
name = str(_trace_message_value(message, "name") or "")
|
||||
if message_type not in {"tool", "toolmessage"} or name != "llmwiki_search":
|
||||
return None
|
||||
content = _trace_message_value(message, "content")
|
||||
if not isinstance(content, str):
|
||||
return None
|
||||
try:
|
||||
payload = json.loads(content)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if not isinstance(payload, dict) or not isinstance(payload.get("results"), list):
|
||||
return None
|
||||
results: list[dict[str, Any]] = []
|
||||
raw_results = payload["results"]
|
||||
for rank, item in enumerate(raw_results[:8], start=1):
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
sections = [
|
||||
section
|
||||
for section in item.get("matched_sections") or []
|
||||
if isinstance(section, dict)
|
||||
]
|
||||
matched_section = item.get("matched_section")
|
||||
if isinstance(matched_section, dict):
|
||||
sections.append(matched_section)
|
||||
section = sections[0] if sections else {}
|
||||
content_text = str(
|
||||
section.get("content")
|
||||
or item.get("content")
|
||||
or item.get("content_markdown")
|
||||
or item.get("summary")
|
||||
or ""
|
||||
)
|
||||
retrieval_mode = str(item.get("retrieval_mode") or "unknown")
|
||||
explicit_vector_score = item.get("vector_score")
|
||||
vector_scores = [
|
||||
float(candidate["score"])
|
||||
for candidate in sections
|
||||
if isinstance(candidate.get("score"), (int, float))
|
||||
]
|
||||
results.append(
|
||||
{
|
||||
"rank": rank,
|
||||
"id": item.get("id") or item.get("wiki_page_id") or item.get("slug"),
|
||||
"kind": "assistant" if item.get("type") == "assistant_knowledge" else "llmwiki",
|
||||
"knowledge_base_id": item.get("knowledge_base_id") or item.get("base_id"),
|
||||
"knowledge_base_name": item.get("knowledge_base_name") or item.get("source"),
|
||||
"title": item.get("title"),
|
||||
"wiki_slug": item.get("wiki_slug") or item.get("slug"),
|
||||
"retrieval_mode": retrieval_mode,
|
||||
"rank_score": item.get("score"),
|
||||
"channel_score": item.get("channel_score"),
|
||||
"vector_score": (
|
||||
explicit_vector_score
|
||||
if isinstance(explicit_vector_score, (int, float))
|
||||
else max(vector_scores)
|
||||
if retrieval_mode == "vector" and vector_scores
|
||||
else None
|
||||
),
|
||||
"content": content_text[:1400],
|
||||
"url": item.get("url") or "",
|
||||
}
|
||||
)
|
||||
tool_call_id = str(_trace_message_value(message, "tool_call_id") or uuid.uuid4())
|
||||
ordinary_ids = [str(item) for item in payload.get("selection") or []]
|
||||
assistant_ids = [str(item) for item in payload.get("assistant_selection") or []]
|
||||
diagnostics = payload.get("diagnostics") if isinstance(payload.get("diagnostics"), dict) else {}
|
||||
selection_count = payload.get("selection_count")
|
||||
if not isinstance(selection_count, int) or isinstance(selection_count, bool):
|
||||
selection_count = len(ordinary_ids) + len(assistant_ids)
|
||||
authorized_space_count = payload.get("authorized_space_count")
|
||||
if not isinstance(authorized_space_count, int) or isinstance(authorized_space_count, bool):
|
||||
# Legacy tool traces only exposed ``knowledge_base_count``. Preserve
|
||||
# those historical records while preferring the exact new field.
|
||||
legacy_count = payload.get("knowledge_base_count")
|
||||
authorized_space_count = legacy_count if isinstance(legacy_count, int) and not isinstance(legacy_count, bool) else 0
|
||||
return {
|
||||
"retrieval_trace_id": str(payload.get("retrieval_trace_id") or f"tool:{tool_call_id}"),
|
||||
"trigger": "tool",
|
||||
"query": str(payload.get("query") or ""),
|
||||
"query_variants": [str(payload.get("query") or "")],
|
||||
"selected_spaces": [
|
||||
*({"kind": "llmwiki", "id": item} for item in ordinary_ids),
|
||||
*({"kind": "assistant", "id": item} for item in assistant_ids),
|
||||
],
|
||||
"selection_count": max(0, selection_count),
|
||||
"authorized_space_count": max(0, authorized_space_count),
|
||||
"mode": str(payload.get("mode") or "empty"),
|
||||
"source_count": int(payload.get("total_result_count") or len(raw_results)),
|
||||
"results_truncated": bool(payload.get("results_truncated")) or len(raw_results) > len(results),
|
||||
"partial": bool(payload.get("partial")),
|
||||
"warnings": [str(item) for item in payload.get("warnings") or []],
|
||||
"fallback_reason": diagnostics.get("fallback_reason"),
|
||||
"fallback_knowledge_base_ids": [
|
||||
str(item) for item in payload.get("fallback_knowledge_base_ids") or []
|
||||
],
|
||||
"diagnostics": diagnostics,
|
||||
"timings_ms": payload.get("timings_ms") if isinstance(payload.get("timings_ms"), dict) else {},
|
||||
"results": results,
|
||||
"zero_result_reason": None if results else payload.get("zero_result_reason"),
|
||||
}
|
||||
|
||||
|
||||
def _extract_knowledge_retrieval_traces(messages: list[Any]) -> list[dict[str, Any]]:
|
||||
"""Extract pre-retrieval and tool traces from persisted message state."""
|
||||
|
||||
traces: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
for message in messages:
|
||||
trace = _trace_message_extra(message).get("knowledge_retrieval_trace")
|
||||
if not isinstance(trace, dict):
|
||||
trace = _tool_trace_from_message(message)
|
||||
if not isinstance(trace, dict):
|
||||
continue
|
||||
item = dict(trace)
|
||||
trace_id = str(item.get("retrieval_trace_id") or "")
|
||||
if not trace_id or trace_id in seen:
|
||||
continue
|
||||
seen.add(trace_id)
|
||||
message_id = _trace_message_value(message, "id")
|
||||
if message_id:
|
||||
item.setdefault("message_id", str(message_id))
|
||||
traces.append(item)
|
||||
return traces
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -929,6 +1074,7 @@ async def get_thread_reference_batches(thread_id: str, request: Request) -> Refe
|
||||
_rag_debug_file_path(),
|
||||
)
|
||||
return ReferenceBatchesResponse(batches=[])
|
||||
retrieval_traces = _extract_knowledge_retrieval_traces(messages)
|
||||
batches_before_display_adapt = build_runtime_reference_batches(messages)
|
||||
raw_batches = batches_before_display_adapt
|
||||
persisted_displays = await _load_persisted_skill_displays()
|
||||
@@ -938,7 +1084,7 @@ async def get_thread_reference_batches(thread_id: str, request: Request) -> Refe
|
||||
_log_rag_debug(
|
||||
"backend reference-batches thread=%s messages=%s turns=%s tool_messages=%s raw_items=%s "
|
||||
"normalized_items=%s kept_items=%s batches=%s batch_sources=%s batch_ids=%s "
|
||||
"display_adapters=%s turns_detail=%s log_file=%s",
|
||||
"display_adapters=%s retrieval_traces=%s turns_detail=%s log_file=%s",
|
||||
sanitize_log_param(thread_id),
|
||||
debug_summary["messages"],
|
||||
debug_summary["turns"],
|
||||
@@ -950,6 +1096,7 @@ async def get_thread_reference_batches(thread_id: str, request: Request) -> Refe
|
||||
",".join(str(batch.get("source_count", 0)) for batch in debug_summary["batches"]),
|
||||
",".join(str(batch.get("id") or "") for batch in debug_summary["batches"]),
|
||||
len(persisted_displays),
|
||||
len(retrieval_traces),
|
||||
json.dumps(debug_summary["turns_detail"], ensure_ascii=False),
|
||||
_rag_debug_file_path(),
|
||||
)
|
||||
@@ -984,7 +1131,8 @@ async def get_thread_reference_batches(thread_id: str, request: Request) -> Refe
|
||||
],
|
||||
)
|
||||
for batch in raw_batches
|
||||
]
|
||||
],
|
||||
retrieval_traces=retrieval_traces,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user