实现RAG参考文献显示模式
This commit is contained in:
1 parent
89bd07be92
commit
f668133297
20 files changed
+1720
-184
No files matched your search
@@ -12,9 +12,12 @@ matching the LangGraph Platform wire format expected by the
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import uuid
|
||||
from typing import Any
|
||||
import logging
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from langgraph.checkpoint.base import empty_checkpoint
|
||||
@@ -23,13 +26,18 @@ from pydantic import BaseModel, Field, field_validator
|
||||
from app.gateway.authz import require_permission
|
||||
from app.gateway.deps import get_checkpointer
|
||||
from app.gateway.utils import sanitize_log_param
|
||||
from deerflow.config.paths import Paths, get_paths
|
||||
from deerflow.runtime import serialize_channel_values
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
from deerflow.utils.time import coerce_iso, now_iso
|
||||
from deerflow.config.paths import Paths, get_paths
|
||||
from deerflow.runtime import serialize_channel_values
|
||||
from deerflow.runtime.references import (
|
||||
build_reference_batches as build_runtime_reference_batches,
|
||||
build_reference_debug_summary,
|
||||
)
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
from deerflow.utils.time import coerce_iso, now_iso
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/api/threads", tags=["threads"])
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/api/threads", tags=["threads"])
|
||||
_RAG_DEBUG_LOGGER_NAME = "rag_citation_debug"
|
||||
|
||||
|
||||
# Metadata keys that the server controls; clients are not allowed to set
|
||||
@@ -41,11 +49,38 @@ router = APIRouter(prefix="/api/threads", tags=["threads"])
|
||||
_SERVER_RESERVED_METADATA_KEYS: frozenset[str] = frozenset({"owner_id", "user_id"})
|
||||
|
||||
|
||||
def _strip_reserved_metadata(metadata: dict[str, Any] | None) -> dict[str, Any]:
|
||||
"""Return ``metadata`` with server-controlled keys removed."""
|
||||
if not metadata:
|
||||
return metadata or {}
|
||||
return {k: v for k, v in metadata.items() if k not in _SERVER_RESERVED_METADATA_KEYS}
|
||||
def _strip_reserved_metadata(metadata: dict[str, Any] | None) -> dict[str, Any]:
|
||||
"""Return ``metadata`` with server-controlled keys removed."""
|
||||
if not metadata:
|
||||
return metadata or {}
|
||||
return {k: v for k, v in metadata.items() if k not in _SERVER_RESERVED_METADATA_KEYS}
|
||||
|
||||
|
||||
def _get_rag_debug_logger() -> logging.Logger:
|
||||
debug_logger = logging.getLogger(_RAG_DEBUG_LOGGER_NAME)
|
||||
if getattr(debug_logger, "_rag_file_configured", False):
|
||||
return debug_logger
|
||||
log_path = Path(os.environ.get("RAG_CITATION_DEBUG_LOG", "logs/rag_citation_debug.log"))
|
||||
if not log_path.is_absolute():
|
||||
log_path = Path.cwd() / log_path
|
||||
log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
handler = logging.FileHandler(log_path, encoding="utf-8")
|
||||
handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
|
||||
debug_logger.addHandler(handler)
|
||||
debug_logger.setLevel(logging.INFO)
|
||||
debug_logger.propagate = False
|
||||
setattr(debug_logger, "_rag_file_configured", True)
|
||||
setattr(debug_logger, "_rag_file_path", str(log_path))
|
||||
return debug_logger
|
||||
|
||||
|
||||
def _rag_debug_file_path() -> str:
|
||||
return str(getattr(_get_rag_debug_logger(), "_rag_file_path", "logs/rag_citation_debug.log"))
|
||||
|
||||
|
||||
def _log_rag_debug(message: str, *args: Any) -> None:
|
||||
logger.info(message, *args)
|
||||
_get_rag_debug_logger().info(message, *args)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -132,11 +167,43 @@ class HistoryEntry(BaseModel):
|
||||
next: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class ThreadHistoryRequest(BaseModel):
|
||||
class ThreadHistoryRequest(BaseModel):
|
||||
"""Request body for checkpoint history."""
|
||||
|
||||
limit: int = Field(default=10, ge=1, le=100, description="Maximum entries")
|
||||
before: str | None = Field(default=None, description="Cursor for pagination")
|
||||
before: str | None = Field(default=None, description="Cursor for pagination")
|
||||
|
||||
|
||||
class ReferenceSourceResponse(BaseModel):
|
||||
index: int
|
||||
id: str | None = None
|
||||
skillName: str = "backend-reference"
|
||||
mode: str = "citation"
|
||||
title: str
|
||||
type: str = "result"
|
||||
snippet: str = ""
|
||||
source: str | None = None
|
||||
time: str | None = None
|
||||
url: str | None = None
|
||||
content: str | None = None
|
||||
score: float | None = None
|
||||
raw: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class ReferenceBatchResponse(BaseModel):
|
||||
id: str
|
||||
assistant_message_id: str
|
||||
sources: list[ReferenceSourceResponse] = Field(default_factory=list)
|
||||
|
||||
|
||||
class ReferenceBatchesResponse(BaseModel):
|
||||
batches: list[ReferenceBatchResponse] = Field(default_factory=list)
|
||||
|
||||
|
||||
class ReferenceDebugLogRequest(BaseModel):
|
||||
event: str
|
||||
thread_id: str | None = None
|
||||
payload: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -163,7 +230,7 @@ def _delete_thread_data(thread_id: str, paths: Paths | None = None, *, user_id:
|
||||
return ThreadDeleteResponse(success=True, message=f"Deleted local thread data for {thread_id}")
|
||||
|
||||
|
||||
def _derive_thread_status(checkpoint_tuple) -> str:
|
||||
def _derive_thread_status(checkpoint_tuple) -> str:
|
||||
"""Derive thread status from checkpoint metadata."""
|
||||
if checkpoint_tuple is None:
|
||||
return "idle"
|
||||
@@ -179,7 +246,251 @@ def _derive_thread_status(checkpoint_tuple) -> str:
|
||||
if tasks:
|
||||
return "interrupted"
|
||||
|
||||
return "idle"
|
||||
return "idle"
|
||||
|
||||
|
||||
def _message_type(message: Any) -> str:
|
||||
if isinstance(message, dict):
|
||||
return str(message.get("type") or message.get("role") or "")
|
||||
return str(getattr(message, "type", "") or "")
|
||||
|
||||
|
||||
def _message_id(message: Any) -> str | None:
|
||||
if isinstance(message, dict):
|
||||
value = message.get("id")
|
||||
else:
|
||||
value = getattr(message, "id", None)
|
||||
return str(value) if value else None
|
||||
|
||||
|
||||
def _message_content(message: Any) -> Any:
|
||||
if isinstance(message, dict):
|
||||
return message.get("content")
|
||||
return getattr(message, "content", None)
|
||||
|
||||
|
||||
def _message_name(message: Any) -> str:
|
||||
if isinstance(message, dict):
|
||||
return str(message.get("name") or (message.get("additional_kwargs") or {}).get("name") or "tool")
|
||||
return str(getattr(message, "name", None) or "tool")
|
||||
|
||||
|
||||
def _has_tool_calls(message: Any) -> bool:
|
||||
if isinstance(message, dict):
|
||||
direct = message.get("tool_calls")
|
||||
chunks = message.get("tool_call_chunks")
|
||||
extra = (message.get("additional_kwargs") or {}).get("tool_calls")
|
||||
else:
|
||||
direct = getattr(message, "tool_calls", None)
|
||||
chunks = getattr(message, "tool_call_chunks", None)
|
||||
extra = (getattr(message, "additional_kwargs", None) or {}).get("tool_calls")
|
||||
return bool(direct or chunks or extra)
|
||||
|
||||
|
||||
def _has_answer_content(message: Any) -> bool:
|
||||
content = _message_content(message)
|
||||
if isinstance(content, str):
|
||||
return bool(content.strip())
|
||||
if isinstance(content, list):
|
||||
for part in content:
|
||||
if isinstance(part, dict) and part.get("type") == "text" and str(part.get("text") or "").strip():
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _extract_tool_payload(message: Any) -> Any:
|
||||
content = _message_content(message)
|
||||
if isinstance(content, dict):
|
||||
return _unwrap_tool_payload(content)
|
||||
if isinstance(content, list):
|
||||
texts: list[str] = []
|
||||
for part in content:
|
||||
if not isinstance(part, dict):
|
||||
continue
|
||||
if isinstance(part.get("json"), dict):
|
||||
return _unwrap_tool_payload(part["json"])
|
||||
if part.get("type") == "json" and isinstance(part.get("json"), dict):
|
||||
return _unwrap_tool_payload(part["json"])
|
||||
if part.get("type") == "text" and isinstance(part.get("text"), str):
|
||||
texts.append(part["text"])
|
||||
unwrapped = _unwrap_tool_payload(part)
|
||||
if unwrapped is not None:
|
||||
return unwrapped
|
||||
content = "\n".join(texts)
|
||||
if isinstance(content, str):
|
||||
return _parse_json_like(content)
|
||||
return None
|
||||
|
||||
|
||||
def _unwrap_tool_payload(value: dict[str, Any]) -> Any:
|
||||
if any(isinstance(value.get(key), list) for key in ("results", "data", "items", "hits", "documents", "records", "sources")):
|
||||
return value
|
||||
for key in ("result", "output", "stdout", "content", "text", "payload", "response"):
|
||||
nested = value.get(key)
|
||||
if isinstance(nested, str):
|
||||
parsed = _parse_json_like(nested)
|
||||
if parsed is not None:
|
||||
return parsed
|
||||
if isinstance(nested, dict):
|
||||
unwrapped = _unwrap_tool_payload(nested)
|
||||
if unwrapped is not None:
|
||||
return unwrapped
|
||||
return value
|
||||
|
||||
|
||||
def _parse_json_like(text: str) -> Any:
|
||||
trimmed = text.strip()
|
||||
if not trimmed:
|
||||
return None
|
||||
try:
|
||||
return json.loads(trimmed)
|
||||
except Exception:
|
||||
pass
|
||||
first_obj = trimmed.find("{")
|
||||
last_obj = trimmed.rfind("}")
|
||||
if 0 <= first_obj < last_obj:
|
||||
try:
|
||||
return json.loads(trimmed[first_obj : last_obj + 1])
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _first_list(payload: Any) -> list[Any]:
|
||||
if isinstance(payload, list):
|
||||
return payload
|
||||
if not isinstance(payload, dict):
|
||||
return []
|
||||
for key in ("results", "data", "items", "hits", "documents", "records", "entries", "matches", "chunks", "sources"):
|
||||
value = payload.get(key)
|
||||
if isinstance(value, list):
|
||||
return value
|
||||
for key in ("result", "payload", "response", "output", "data"):
|
||||
nested = payload.get(key)
|
||||
found = _first_list(nested)
|
||||
if found:
|
||||
return found
|
||||
return []
|
||||
|
||||
|
||||
def _nested_records(raw: Any) -> list[dict[str, Any]]:
|
||||
if not isinstance(raw, dict):
|
||||
return []
|
||||
layers = [raw]
|
||||
for key in ("metadata", "meta", "attrs", "attributes", "properties", "extra"):
|
||||
nested = raw.get(key)
|
||||
if isinstance(nested, dict):
|
||||
layers.append(nested)
|
||||
return layers
|
||||
|
||||
|
||||
def _pick(layers: list[dict[str, Any]], keys: tuple[str, ...]) -> str | None:
|
||||
for layer in layers:
|
||||
for key in keys:
|
||||
value = layer.get(key)
|
||||
if isinstance(value, (str, int, float)) and str(value).strip():
|
||||
return str(value).strip()
|
||||
return None
|
||||
|
||||
|
||||
def _normalize_reference_item(raw: Any, fallback_index: int) -> dict[str, Any] | None:
|
||||
if not isinstance(raw, dict):
|
||||
return None
|
||||
layers = _nested_records(raw)
|
||||
title = _pick(layers, ("title", "m_title", "name", "question", "keyword", "query", "text"))
|
||||
content = _pick(layers, ("content", "page_content", "summary", "snippet", "description", "content_preview", "abstract", "body"))
|
||||
url = _pick(layers, ("url", "link", "href", "source_url", "web_url"))
|
||||
source = _pick(layers, ("source", "source1", "site", "provider", "from"))
|
||||
time = _pick(layers, ("time", "date", "published_date", "publishedDate", "m_publish", "publish_time", "created_at"))
|
||||
item_id = _pick(layers, ("recUuid", "rec_uuid", "uuid", "docId", "doc_id", "id", "recordId", "taskId"))
|
||||
score_raw = _pick(layers, ("score", "relevance", "similarity"))
|
||||
score = None
|
||||
if score_raw is not None:
|
||||
try:
|
||||
score = float(score_raw)
|
||||
except Exception:
|
||||
score = None
|
||||
if not title:
|
||||
title = source or f"结果 {fallback_index}"
|
||||
if not content:
|
||||
content = ""
|
||||
if not title and not content and not url:
|
||||
return None
|
||||
return {
|
||||
"id": item_id or url,
|
||||
"title": title,
|
||||
"type": "result",
|
||||
"snippet": content,
|
||||
"content": content,
|
||||
"source": source,
|
||||
"time": time,
|
||||
"url": url,
|
||||
"score": score,
|
||||
"raw": raw,
|
||||
}
|
||||
|
||||
|
||||
def _reference_key(source: dict[str, Any]) -> str:
|
||||
raw = source.get("raw") if isinstance(source.get("raw"), dict) else {}
|
||||
item_id = (
|
||||
source.get("id")
|
||||
or source.get("url")
|
||||
or raw.get("recUuid")
|
||||
or raw.get("uuid")
|
||||
or raw.get("docId")
|
||||
or raw.get("id")
|
||||
)
|
||||
if item_id:
|
||||
return str(item_id).casefold().strip()
|
||||
return "|".join(
|
||||
str(source.get(key) or "").casefold().strip()
|
||||
for key in ("title", "source", "time", "content", "snippet")
|
||||
)
|
||||
|
||||
|
||||
def _build_reference_batches(messages: list[Any]) -> list[ReferenceBatchResponse]:
|
||||
batches: list[ReferenceBatchResponse] = []
|
||||
for index, message in enumerate(messages):
|
||||
if _message_type(message) != "human":
|
||||
continue
|
||||
end = len(messages)
|
||||
for cursor in range(index + 1, len(messages)):
|
||||
if _message_type(messages[cursor]) == "human":
|
||||
end = cursor
|
||||
break
|
||||
assistant_message: Any | None = None
|
||||
for cursor in range(end - 1, index, -1):
|
||||
candidate = messages[cursor]
|
||||
if (
|
||||
_message_type(candidate) == "ai"
|
||||
and _message_id(candidate)
|
||||
and _has_answer_content(candidate)
|
||||
and not _has_tool_calls(candidate)
|
||||
):
|
||||
assistant_message = candidate
|
||||
break
|
||||
if assistant_message is None:
|
||||
continue
|
||||
seen: set[str] = set()
|
||||
sources: list[ReferenceSourceResponse] = []
|
||||
for tool_message in messages[index + 1 : end]:
|
||||
if _message_type(tool_message) != "tool":
|
||||
continue
|
||||
payload = _extract_tool_payload(tool_message)
|
||||
for raw_item in _first_list(payload):
|
||||
normalized = _normalize_reference_item(raw_item, len(sources) + 1)
|
||||
if not normalized:
|
||||
continue
|
||||
key = _reference_key(normalized)
|
||||
if key and key in seen:
|
||||
continue
|
||||
if key:
|
||||
seen.add(key)
|
||||
sources.append(ReferenceSourceResponse(index=len(sources) + 1, **normalized))
|
||||
if sources:
|
||||
mid = _message_id(assistant_message) or str(index)
|
||||
batches.append(ReferenceBatchResponse(id=mid, assistant_message_id=mid, sources=sources))
|
||||
return batches
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -406,9 +717,9 @@ async def get_thread(thread_id: str, request: Request) -> ThreadResponse:
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@router.get("/{thread_id}/state", response_model=ThreadStateResponse)
|
||||
@require_permission("threads", "read", owner_check=True)
|
||||
async def get_thread_state(thread_id: str, request: Request) -> ThreadStateResponse:
|
||||
@router.get("/{thread_id}/state", response_model=ThreadStateResponse)
|
||||
@require_permission("threads", "read", owner_check=True)
|
||||
async def get_thread_state(thread_id: str, request: Request) -> ThreadStateResponse:
|
||||
"""Get the latest state snapshot for a thread.
|
||||
|
||||
Channel values are serialized to ensure LangChain message objects
|
||||
@@ -455,12 +766,84 @@ async def get_thread_state(thread_id: str, request: Request) -> ThreadStateRespo
|
||||
parent_checkpoint_id=parent_checkpoint_id,
|
||||
created_at=coerce_iso(metadata.get("created_at", "")),
|
||||
tasks=tasks,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/{thread_id}/state", response_model=ThreadStateResponse)
|
||||
@require_permission("threads", "write", owner_check=True, require_existing=True)
|
||||
async def update_thread_state(thread_id: str, body: ThreadStateUpdateRequest, request: Request) -> ThreadStateResponse:
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{thread_id}/reference-batches", response_model=ReferenceBatchesResponse)
|
||||
@require_permission("threads", "read", owner_check=True)
|
||||
async def get_thread_reference_batches(thread_id: str, request: Request) -> ReferenceBatchesResponse:
|
||||
"""Return backend-built citation batches for completed assistant turns."""
|
||||
checkpointer = get_checkpointer(request)
|
||||
config = {"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}}
|
||||
try:
|
||||
checkpoint_tuple = await checkpointer.aget_tuple(config)
|
||||
except Exception:
|
||||
logger.exception("Failed to get reference batches for thread %s", sanitize_log_param(thread_id))
|
||||
raise HTTPException(status_code=500, detail="Failed to get reference batches")
|
||||
|
||||
if checkpoint_tuple is None:
|
||||
raise HTTPException(status_code=404, detail=f"Thread {thread_id} not found")
|
||||
|
||||
checkpoint = getattr(checkpoint_tuple, "checkpoint", {}) or {}
|
||||
values = serialize_channel_values(checkpoint.get("channel_values", {}) or {})
|
||||
messages = values.get("messages")
|
||||
if not isinstance(messages, list):
|
||||
_log_rag_debug(
|
||||
"backend reference-batches thread=%s messages=missing batches=0 log_file=%s",
|
||||
sanitize_log_param(thread_id),
|
||||
_rag_debug_file_path(),
|
||||
)
|
||||
return ReferenceBatchesResponse(batches=[])
|
||||
raw_batches = build_runtime_reference_batches(messages)
|
||||
debug_summary = build_reference_debug_summary(messages, raw_batches)
|
||||
_log_rag_debug(
|
||||
"backend reference-batches thread=%s messages=%s turns=%s tool_messages=%s raw_items=%s "
|
||||
"normalized_items=%s kept_items=%s batches=%s batch_sources=%s batch_ids=%s turns_detail=%s log_file=%s",
|
||||
sanitize_log_param(thread_id),
|
||||
debug_summary["messages"],
|
||||
debug_summary["turns"],
|
||||
debug_summary["tool_messages"],
|
||||
debug_summary["raw_items"],
|
||||
debug_summary["normalized_items"],
|
||||
debug_summary["kept_items"],
|
||||
len(raw_batches),
|
||||
",".join(str(batch.get("source_count", 0)) for batch in debug_summary["batches"]),
|
||||
",".join(str(batch.get("id") or "") for batch in debug_summary["batches"]),
|
||||
json.dumps(debug_summary["turns_detail"], ensure_ascii=False),
|
||||
_rag_debug_file_path(),
|
||||
)
|
||||
return ReferenceBatchesResponse(
|
||||
batches=[
|
||||
ReferenceBatchResponse(
|
||||
id=str(batch.get("id") or ""),
|
||||
assistant_message_id=str(batch.get("assistant_message_id") or batch.get("id") or ""),
|
||||
sources=[
|
||||
ReferenceSourceResponse(**source)
|
||||
for source in batch.get("sources", [])
|
||||
if isinstance(source, dict)
|
||||
],
|
||||
)
|
||||
for batch in raw_batches
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@router.post("/reference-debug-log")
|
||||
async def write_reference_debug_log(body: ReferenceDebugLogRequest) -> dict[str, str | bool]:
|
||||
"""Append frontend citation diagnostics to the backend RAG debug log."""
|
||||
_get_rag_debug_logger().info(
|
||||
"frontend %s thread=%s payload=%s log_file=%s",
|
||||
body.event,
|
||||
sanitize_log_param(body.thread_id or ""),
|
||||
json.dumps(body.payload, ensure_ascii=False, default=str),
|
||||
_rag_debug_file_path(),
|
||||
)
|
||||
return {"ok": True, "log_file": _rag_debug_file_path()}
|
||||
|
||||
|
||||
@router.post("/{thread_id}/state", response_model=ThreadStateResponse)
|
||||
@require_permission("threads", "write", owner_check=True, require_existing=True)
|
||||
async def update_thread_state(thread_id: str, body: ThreadStateUpdateRequest, request: Request) -> ThreadStateResponse:
|
||||
"""Update thread state (e.g. for human-in-the-loop resume or title rename).
|
||||
|
||||
Writes a new checkpoint that merges *body.values* into the latest
|
||||
|
||||
Reference in new issue
Block a user