实现RAG参考文献显示模式

This commit is contained in:
吉鑫升 authored and xuhd committed 2026-05-28 18:28:57 +08:00
1 parent 89bd07be92
commit f668133297
20 files changed
+1720 -184

No files matched your search

@@ -12,9 +12,12 @@ matching the LangGraph Platform wire format expected by the
from __future__ import annotations
import logging
import uuid
from typing import Any
import logging
import json
import os
from pathlib import Path
import uuid
from typing import Any
from fastapi import APIRouter, HTTPException, Request
from langgraph.checkpoint.base import empty_checkpoint
@@ -23,13 +26,18 @@ from pydantic import BaseModel, Field, field_validator
from app.gateway.authz import require_permission
from app.gateway.deps import get_checkpointer
from app.gateway.utils import sanitize_log_param
from deerflow.config.paths import Paths, get_paths
from deerflow.runtime import serialize_channel_values
from deerflow.runtime.user_context import get_effective_user_id
from deerflow.utils.time import coerce_iso, now_iso
from deerflow.config.paths import Paths, get_paths
from deerflow.runtime import serialize_channel_values
from deerflow.runtime.references import (
build_reference_batches as build_runtime_reference_batches,
build_reference_debug_summary,
)
from deerflow.runtime.user_context import get_effective_user_id
from deerflow.utils.time import coerce_iso, now_iso
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/threads", tags=["threads"])
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/threads", tags=["threads"])
_RAG_DEBUG_LOGGER_NAME = "rag_citation_debug"
# Metadata keys that the server controls; clients are not allowed to set
@@ -41,11 +49,38 @@ router = APIRouter(prefix="/api/threads", tags=["threads"])
_SERVER_RESERVED_METADATA_KEYS: frozenset[str] = frozenset({"owner_id", "user_id"})
def _strip_reserved_metadata(metadata: dict[str, Any] | None) -> dict[str, Any]:
"""Return ``metadata`` with server-controlled keys removed."""
if not metadata:
return metadata or {}
return {k: v for k, v in metadata.items() if k not in _SERVER_RESERVED_METADATA_KEYS}
def _strip_reserved_metadata(metadata: dict[str, Any] | None) -> dict[str, Any]:
"""Return ``metadata`` with server-controlled keys removed."""
if not metadata:
return metadata or {}
return {k: v for k, v in metadata.items() if k not in _SERVER_RESERVED_METADATA_KEYS}
def _get_rag_debug_logger() -> logging.Logger:
debug_logger = logging.getLogger(_RAG_DEBUG_LOGGER_NAME)
if getattr(debug_logger, "_rag_file_configured", False):
return debug_logger
log_path = Path(os.environ.get("RAG_CITATION_DEBUG_LOG", "logs/rag_citation_debug.log"))
if not log_path.is_absolute():
log_path = Path.cwd() / log_path
log_path.parent.mkdir(parents=True, exist_ok=True)
handler = logging.FileHandler(log_path, encoding="utf-8")
handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
debug_logger.addHandler(handler)
debug_logger.setLevel(logging.INFO)
debug_logger.propagate = False
setattr(debug_logger, "_rag_file_configured", True)
setattr(debug_logger, "_rag_file_path", str(log_path))
return debug_logger
def _rag_debug_file_path() -> str:
return str(getattr(_get_rag_debug_logger(), "_rag_file_path", "logs/rag_citation_debug.log"))
def _log_rag_debug(message: str, *args: Any) -> None:
logger.info(message, *args)
_get_rag_debug_logger().info(message, *args)
# ---------------------------------------------------------------------------
@@ -132,11 +167,43 @@ class HistoryEntry(BaseModel):
next: list[str] = Field(default_factory=list)
class ThreadHistoryRequest(BaseModel):
class ThreadHistoryRequest(BaseModel):
"""Request body for checkpoint history."""
limit: int = Field(default=10, ge=1, le=100, description="Maximum entries")
before: str | None = Field(default=None, description="Cursor for pagination")
before: str | None = Field(default=None, description="Cursor for pagination")
class ReferenceSourceResponse(BaseModel):
index: int
id: str | None = None
skillName: str = "backend-reference"
mode: str = "citation"
title: str
type: str = "result"
snippet: str = ""
source: str | None = None
time: str | None = None
url: str | None = None
content: str | None = None
score: float | None = None
raw: dict[str, Any] = Field(default_factory=dict)
class ReferenceBatchResponse(BaseModel):
id: str
assistant_message_id: str
sources: list[ReferenceSourceResponse] = Field(default_factory=list)
class ReferenceBatchesResponse(BaseModel):
batches: list[ReferenceBatchResponse] = Field(default_factory=list)
class ReferenceDebugLogRequest(BaseModel):
event: str
thread_id: str | None = None
payload: dict[str, Any] = Field(default_factory=dict)
# ---------------------------------------------------------------------------
@@ -163,7 +230,7 @@ def _delete_thread_data(thread_id: str, paths: Paths | None = None, *, user_id:
return ThreadDeleteResponse(success=True, message=f"Deleted local thread data for {thread_id}")
def _derive_thread_status(checkpoint_tuple) -> str:
def _derive_thread_status(checkpoint_tuple) -> str:
"""Derive thread status from checkpoint metadata."""
if checkpoint_tuple is None:
return "idle"
@@ -179,7 +246,251 @@ def _derive_thread_status(checkpoint_tuple) -> str:
if tasks:
return "interrupted"
return "idle"
return "idle"
def _message_type(message: Any) -> str:
if isinstance(message, dict):
return str(message.get("type") or message.get("role") or "")
return str(getattr(message, "type", "") or "")
def _message_id(message: Any) -> str | None:
if isinstance(message, dict):
value = message.get("id")
else:
value = getattr(message, "id", None)
return str(value) if value else None
def _message_content(message: Any) -> Any:
if isinstance(message, dict):
return message.get("content")
return getattr(message, "content", None)
def _message_name(message: Any) -> str:
if isinstance(message, dict):
return str(message.get("name") or (message.get("additional_kwargs") or {}).get("name") or "tool")
return str(getattr(message, "name", None) or "tool")
def _has_tool_calls(message: Any) -> bool:
if isinstance(message, dict):
direct = message.get("tool_calls")
chunks = message.get("tool_call_chunks")
extra = (message.get("additional_kwargs") or {}).get("tool_calls")
else:
direct = getattr(message, "tool_calls", None)
chunks = getattr(message, "tool_call_chunks", None)
extra = (getattr(message, "additional_kwargs", None) or {}).get("tool_calls")
return bool(direct or chunks or extra)
def _has_answer_content(message: Any) -> bool:
content = _message_content(message)
if isinstance(content, str):
return bool(content.strip())
if isinstance(content, list):
for part in content:
if isinstance(part, dict) and part.get("type") == "text" and str(part.get("text") or "").strip():
return True
return False
def _extract_tool_payload(message: Any) -> Any:
content = _message_content(message)
if isinstance(content, dict):
return _unwrap_tool_payload(content)
if isinstance(content, list):
texts: list[str] = []
for part in content:
if not isinstance(part, dict):
continue
if isinstance(part.get("json"), dict):
return _unwrap_tool_payload(part["json"])
if part.get("type") == "json" and isinstance(part.get("json"), dict):
return _unwrap_tool_payload(part["json"])
if part.get("type") == "text" and isinstance(part.get("text"), str):
texts.append(part["text"])
unwrapped = _unwrap_tool_payload(part)
if unwrapped is not None:
return unwrapped
content = "\n".join(texts)
if isinstance(content, str):
return _parse_json_like(content)
return None
def _unwrap_tool_payload(value: dict[str, Any]) -> Any:
if any(isinstance(value.get(key), list) for key in ("results", "data", "items", "hits", "documents", "records", "sources")):
return value
for key in ("result", "output", "stdout", "content", "text", "payload", "response"):
nested = value.get(key)
if isinstance(nested, str):
parsed = _parse_json_like(nested)
if parsed is not None:
return parsed
if isinstance(nested, dict):
unwrapped = _unwrap_tool_payload(nested)
if unwrapped is not None:
return unwrapped
return value
def _parse_json_like(text: str) -> Any:
trimmed = text.strip()
if not trimmed:
return None
try:
return json.loads(trimmed)
except Exception:
pass
first_obj = trimmed.find("{")
last_obj = trimmed.rfind("}")
if 0 <= first_obj < last_obj:
try:
return json.loads(trimmed[first_obj : last_obj + 1])
except Exception:
return None
return None
def _first_list(payload: Any) -> list[Any]:
if isinstance(payload, list):
return payload
if not isinstance(payload, dict):
return []
for key in ("results", "data", "items", "hits", "documents", "records", "entries", "matches", "chunks", "sources"):
value = payload.get(key)
if isinstance(value, list):
return value
for key in ("result", "payload", "response", "output", "data"):
nested = payload.get(key)
found = _first_list(nested)
if found:
return found
return []
def _nested_records(raw: Any) -> list[dict[str, Any]]:
if not isinstance(raw, dict):
return []
layers = [raw]
for key in ("metadata", "meta", "attrs", "attributes", "properties", "extra"):
nested = raw.get(key)
if isinstance(nested, dict):
layers.append(nested)
return layers
def _pick(layers: list[dict[str, Any]], keys: tuple[str, ...]) -> str | None:
for layer in layers:
for key in keys:
value = layer.get(key)
if isinstance(value, (str, int, float)) and str(value).strip():
return str(value).strip()
return None
def _normalize_reference_item(raw: Any, fallback_index: int) -> dict[str, Any] | None:
if not isinstance(raw, dict):
return None
layers = _nested_records(raw)
title = _pick(layers, ("title", "m_title", "name", "question", "keyword", "query", "text"))
content = _pick(layers, ("content", "page_content", "summary", "snippet", "description", "content_preview", "abstract", "body"))
url = _pick(layers, ("url", "link", "href", "source_url", "web_url"))
source = _pick(layers, ("source", "source1", "site", "provider", "from"))
time = _pick(layers, ("time", "date", "published_date", "publishedDate", "m_publish", "publish_time", "created_at"))
item_id = _pick(layers, ("recUuid", "rec_uuid", "uuid", "docId", "doc_id", "id", "recordId", "taskId"))
score_raw = _pick(layers, ("score", "relevance", "similarity"))
score = None
if score_raw is not None:
try:
score = float(score_raw)
except Exception:
score = None
if not title:
title = source or f"结果 {fallback_index}"
if not content:
content = ""
if not title and not content and not url:
return None
return {
"id": item_id or url,
"title": title,
"type": "result",
"snippet": content,
"content": content,
"source": source,
"time": time,
"url": url,
"score": score,
"raw": raw,
}
def _reference_key(source: dict[str, Any]) -> str:
raw = source.get("raw") if isinstance(source.get("raw"), dict) else {}
item_id = (
source.get("id")
or source.get("url")
or raw.get("recUuid")
or raw.get("uuid")
or raw.get("docId")
or raw.get("id")
)
if item_id:
return str(item_id).casefold().strip()
return "|".join(
str(source.get(key) or "").casefold().strip()
for key in ("title", "source", "time", "content", "snippet")
)
def _build_reference_batches(messages: list[Any]) -> list[ReferenceBatchResponse]:
batches: list[ReferenceBatchResponse] = []
for index, message in enumerate(messages):
if _message_type(message) != "human":
continue
end = len(messages)
for cursor in range(index + 1, len(messages)):
if _message_type(messages[cursor]) == "human":
end = cursor
break
assistant_message: Any | None = None
for cursor in range(end - 1, index, -1):
candidate = messages[cursor]
if (
_message_type(candidate) == "ai"
and _message_id(candidate)
and _has_answer_content(candidate)
and not _has_tool_calls(candidate)
):
assistant_message = candidate
break
if assistant_message is None:
continue
seen: set[str] = set()
sources: list[ReferenceSourceResponse] = []
for tool_message in messages[index + 1 : end]:
if _message_type(tool_message) != "tool":
continue
payload = _extract_tool_payload(tool_message)
for raw_item in _first_list(payload):
normalized = _normalize_reference_item(raw_item, len(sources) + 1)
if not normalized:
continue
key = _reference_key(normalized)
if key and key in seen:
continue
if key:
seen.add(key)
sources.append(ReferenceSourceResponse(index=len(sources) + 1, **normalized))
if sources:
mid = _message_id(assistant_message) or str(index)
batches.append(ReferenceBatchResponse(id=mid, assistant_message_id=mid, sources=sources))
return batches
# ---------------------------------------------------------------------------
@@ -406,9 +717,9 @@ async def get_thread(thread_id: str, request: Request) -> ThreadResponse:
# ---------------------------------------------------------------------------
@router.get("/{thread_id}/state", response_model=ThreadStateResponse)
@require_permission("threads", "read", owner_check=True)
async def get_thread_state(thread_id: str, request: Request) -> ThreadStateResponse:
@router.get("/{thread_id}/state", response_model=ThreadStateResponse)
@require_permission("threads", "read", owner_check=True)
async def get_thread_state(thread_id: str, request: Request) -> ThreadStateResponse:
"""Get the latest state snapshot for a thread.
Channel values are serialized to ensure LangChain message objects
@@ -455,12 +766,84 @@ async def get_thread_state(thread_id: str, request: Request) -> ThreadStateRespo
parent_checkpoint_id=parent_checkpoint_id,
created_at=coerce_iso(metadata.get("created_at", "")),
tasks=tasks,
)
@router.post("/{thread_id}/state", response_model=ThreadStateResponse)
@require_permission("threads", "write", owner_check=True, require_existing=True)
async def update_thread_state(thread_id: str, body: ThreadStateUpdateRequest, request: Request) -> ThreadStateResponse:
)
@router.get("/{thread_id}/reference-batches", response_model=ReferenceBatchesResponse)
@require_permission("threads", "read", owner_check=True)
async def get_thread_reference_batches(thread_id: str, request: Request) -> ReferenceBatchesResponse:
"""Return backend-built citation batches for completed assistant turns."""
checkpointer = get_checkpointer(request)
config = {"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}}
try:
checkpoint_tuple = await checkpointer.aget_tuple(config)
except Exception:
logger.exception("Failed to get reference batches for thread %s", sanitize_log_param(thread_id))
raise HTTPException(status_code=500, detail="Failed to get reference batches")
if checkpoint_tuple is None:
raise HTTPException(status_code=404, detail=f"Thread {thread_id} not found")
checkpoint = getattr(checkpoint_tuple, "checkpoint", {}) or {}
values = serialize_channel_values(checkpoint.get("channel_values", {}) or {})
messages = values.get("messages")
if not isinstance(messages, list):
_log_rag_debug(
"backend reference-batches thread=%s messages=missing batches=0 log_file=%s",
sanitize_log_param(thread_id),
_rag_debug_file_path(),
)
return ReferenceBatchesResponse(batches=[])
raw_batches = build_runtime_reference_batches(messages)
debug_summary = build_reference_debug_summary(messages, raw_batches)
_log_rag_debug(
"backend reference-batches thread=%s messages=%s turns=%s tool_messages=%s raw_items=%s "
"normalized_items=%s kept_items=%s batches=%s batch_sources=%s batch_ids=%s turns_detail=%s log_file=%s",
sanitize_log_param(thread_id),
debug_summary["messages"],
debug_summary["turns"],
debug_summary["tool_messages"],
debug_summary["raw_items"],
debug_summary["normalized_items"],
debug_summary["kept_items"],
len(raw_batches),
",".join(str(batch.get("source_count", 0)) for batch in debug_summary["batches"]),
",".join(str(batch.get("id") or "") for batch in debug_summary["batches"]),
json.dumps(debug_summary["turns_detail"], ensure_ascii=False),
_rag_debug_file_path(),
)
return ReferenceBatchesResponse(
batches=[
ReferenceBatchResponse(
id=str(batch.get("id") or ""),
assistant_message_id=str(batch.get("assistant_message_id") or batch.get("id") or ""),
sources=[
ReferenceSourceResponse(**source)
for source in batch.get("sources", [])
if isinstance(source, dict)
],
)
for batch in raw_batches
]
)
@router.post("/reference-debug-log")
async def write_reference_debug_log(body: ReferenceDebugLogRequest) -> dict[str, str | bool]:
"""Append frontend citation diagnostics to the backend RAG debug log."""
_get_rag_debug_logger().info(
"frontend %s thread=%s payload=%s log_file=%s",
body.event,
sanitize_log_param(body.thread_id or ""),
json.dumps(body.payload, ensure_ascii=False, default=str),
_rag_debug_file_path(),
)
return {"ok": True, "log_file": _rag_debug_file_path()}
@router.post("/{thread_id}/state", response_model=ThreadStateResponse)
@require_permission("threads", "write", owner_check=True, require_existing=True)
async def update_thread_state(thread_id: str, body: ThreadStateUpdateRequest, request: Request) -> ThreadStateResponse:
"""Update thread state (e.g. for human-in-the-loop resume or title rename).
Writes a new checkpoint that merges *body.values* into the latest