sync update from LAN repo (source ji @ 85078a31)

This commit is contained in:
gozeros committed 2026-10-21 15:09:16 +08:00
1 parent 5d6627af50
commit e602d88695
864 files changed
+99150 -6346

No files matched your search

@@ -29,14 +29,14 @@ from app.gateway.authz import require_permission
from app.gateway.deps import get_checkpointer
from app.gateway.llmwiki_deposit import delete_conversation_deposit_documents
from app.gateway.utils import sanitize_log_param
from deerflow.config.extensions_config import SkillDisplayConfig
from deerflow.config.paths import Paths, get_paths
from deerflow.persistence.engine import get_engine, get_session_factory
from deerflow.persistence.skills.model import SkillDisplayAdapterRow
from deerflow.agents.roundtable_orchestrator.delivery import (
classify_seat_delivery,
visible_seat_text,
)
from deerflow.config.extensions_config import SkillDisplayConfig
from deerflow.config.paths import Paths, get_paths
from deerflow.persistence.engine import get_engine, get_session_factory
from deerflow.persistence.skills.model import SkillDisplayAdapterRow
from deerflow.runtime import serialize_channel_values
from deerflow.runtime.references import (
adapt_reference_batches_with_displays,
@@ -272,6 +272,7 @@ class ReferenceBatchResponse(BaseModel):
class ReferenceBatchesResponse(BaseModel):
batches: list[ReferenceBatchResponse] = Field(default_factory=list)
retrieval_traces: list[dict[str, Any]] = Field(default_factory=list)
class ReferenceDebugLogRequest(BaseModel):
@@ -280,6 +281,150 @@ class ReferenceDebugLogRequest(BaseModel):
payload: dict[str, Any] = Field(default_factory=dict)
def _trace_message_extra(message: Any) -> dict[str, Any]:
if isinstance(message, dict):
value = message.get("additional_kwargs") or message.get("additionalKwargs")
else:
value = getattr(message, "additional_kwargs", None)
return value if isinstance(value, dict) else {}
def _trace_message_value(message: Any, key: str) -> Any:
if isinstance(message, dict):
return message.get(key)
return getattr(message, key, None)
def _tool_trace_from_message(message: Any) -> dict[str, Any] | None:
"""Build a bounded trace for a model-driven ``llmwiki_search`` call."""
message_type = str(_trace_message_value(message, "type") or "").lower()
name = str(_trace_message_value(message, "name") or "")
if message_type not in {"tool", "toolmessage"} or name != "llmwiki_search":
return None
content = _trace_message_value(message, "content")
if not isinstance(content, str):
return None
try:
payload = json.loads(content)
except (TypeError, ValueError):
return None
if not isinstance(payload, dict) or not isinstance(payload.get("results"), list):
return None
results: list[dict[str, Any]] = []
raw_results = payload["results"]
for rank, item in enumerate(raw_results[:8], start=1):
if not isinstance(item, dict):
continue
sections = [
section
for section in item.get("matched_sections") or []
if isinstance(section, dict)
]
matched_section = item.get("matched_section")
if isinstance(matched_section, dict):
sections.append(matched_section)
section = sections[0] if sections else {}
content_text = str(
section.get("content")
or item.get("content")
or item.get("content_markdown")
or item.get("summary")
or ""
)
retrieval_mode = str(item.get("retrieval_mode") or "unknown")
explicit_vector_score = item.get("vector_score")
vector_scores = [
float(candidate["score"])
for candidate in sections
if isinstance(candidate.get("score"), (int, float))
]
results.append(
{
"rank": rank,
"id": item.get("id") or item.get("wiki_page_id") or item.get("slug"),
"kind": "assistant" if item.get("type") == "assistant_knowledge" else "llmwiki",
"knowledge_base_id": item.get("knowledge_base_id") or item.get("base_id"),
"knowledge_base_name": item.get("knowledge_base_name") or item.get("source"),
"title": item.get("title"),
"wiki_slug": item.get("wiki_slug") or item.get("slug"),
"retrieval_mode": retrieval_mode,
"rank_score": item.get("score"),
"channel_score": item.get("channel_score"),
"vector_score": (
explicit_vector_score
if isinstance(explicit_vector_score, (int, float))
else max(vector_scores)
if retrieval_mode == "vector" and vector_scores
else None
),
"content": content_text[:1400],
"url": item.get("url") or "",
}
)
tool_call_id = str(_trace_message_value(message, "tool_call_id") or uuid.uuid4())
ordinary_ids = [str(item) for item in payload.get("selection") or []]
assistant_ids = [str(item) for item in payload.get("assistant_selection") or []]
diagnostics = payload.get("diagnostics") if isinstance(payload.get("diagnostics"), dict) else {}
selection_count = payload.get("selection_count")
if not isinstance(selection_count, int) or isinstance(selection_count, bool):
selection_count = len(ordinary_ids) + len(assistant_ids)
authorized_space_count = payload.get("authorized_space_count")
if not isinstance(authorized_space_count, int) or isinstance(authorized_space_count, bool):
# Legacy tool traces only exposed ``knowledge_base_count``. Preserve
# those historical records while preferring the exact new field.
legacy_count = payload.get("knowledge_base_count")
authorized_space_count = legacy_count if isinstance(legacy_count, int) and not isinstance(legacy_count, bool) else 0
return {
"retrieval_trace_id": str(payload.get("retrieval_trace_id") or f"tool:{tool_call_id}"),
"trigger": "tool",
"query": str(payload.get("query") or ""),
"query_variants": [str(payload.get("query") or "")],
"selected_spaces": [
*({"kind": "llmwiki", "id": item} for item in ordinary_ids),
*({"kind": "assistant", "id": item} for item in assistant_ids),
],
"selection_count": max(0, selection_count),
"authorized_space_count": max(0, authorized_space_count),
"mode": str(payload.get("mode") or "empty"),
"source_count": int(payload.get("total_result_count") or len(raw_results)),
"results_truncated": bool(payload.get("results_truncated")) or len(raw_results) > len(results),
"partial": bool(payload.get("partial")),
"warnings": [str(item) for item in payload.get("warnings") or []],
"fallback_reason": diagnostics.get("fallback_reason"),
"fallback_knowledge_base_ids": [
str(item) for item in payload.get("fallback_knowledge_base_ids") or []
],
"diagnostics": diagnostics,
"timings_ms": payload.get("timings_ms") if isinstance(payload.get("timings_ms"), dict) else {},
"results": results,
"zero_result_reason": None if results else payload.get("zero_result_reason"),
}
def _extract_knowledge_retrieval_traces(messages: list[Any]) -> list[dict[str, Any]]:
"""Extract pre-retrieval and tool traces from persisted message state."""
traces: list[dict[str, Any]] = []
seen: set[str] = set()
for message in messages:
trace = _trace_message_extra(message).get("knowledge_retrieval_trace")
if not isinstance(trace, dict):
trace = _tool_trace_from_message(message)
if not isinstance(trace, dict):
continue
item = dict(trace)
trace_id = str(item.get("retrieval_trace_id") or "")
if not trace_id or trace_id in seen:
continue
seen.add(trace_id)
message_id = _trace_message_value(message, "id")
if message_id:
item.setdefault("message_id", str(message_id))
traces.append(item)
return traces
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
@@ -929,6 +1074,7 @@ async def get_thread_reference_batches(thread_id: str, request: Request) -> Refe
_rag_debug_file_path(),
)
return ReferenceBatchesResponse(batches=[])
retrieval_traces = _extract_knowledge_retrieval_traces(messages)
batches_before_display_adapt = build_runtime_reference_batches(messages)
raw_batches = batches_before_display_adapt
persisted_displays = await _load_persisted_skill_displays()
@@ -938,7 +1084,7 @@ async def get_thread_reference_batches(thread_id: str, request: Request) -> Refe
_log_rag_debug(
"backend reference-batches thread=%s messages=%s turns=%s tool_messages=%s raw_items=%s "
"normalized_items=%s kept_items=%s batches=%s batch_sources=%s batch_ids=%s "
"display_adapters=%s turns_detail=%s log_file=%s",
"display_adapters=%s retrieval_traces=%s turns_detail=%s log_file=%s",
sanitize_log_param(thread_id),
debug_summary["messages"],
debug_summary["turns"],
@@ -950,6 +1096,7 @@ async def get_thread_reference_batches(thread_id: str, request: Request) -> Refe
",".join(str(batch.get("source_count", 0)) for batch in debug_summary["batches"]),
",".join(str(batch.get("id") or "") for batch in debug_summary["batches"]),
len(persisted_displays),
len(retrieval_traces),
json.dumps(debug_summary["turns_detail"], ensure_ascii=False),
_rag_debug_file_path(),
)
@@ -984,7 +1131,8 @@ async def get_thread_reference_batches(thread_id: str, request: Request) -> Refe
],
)
for batch in raw_batches
]
],
retrieval_traces=retrieval_traces,
)