sync snapshot from LAN repo (source codex/kb-context-retrieval-20261021 @ b69568e4)

This commit is contained in:
gozeros committed 2026-11-05 11:03:53 +08:00
1 parent ea9dcff9c0
commit 4cf657f725
781 files changed
+13559 -104852

No files matched your search

@@ -17,7 +17,6 @@ import json
import logging
import os
import uuid
from contextlib import aclosing
from pathlib import Path
from typing import Any
@@ -27,7 +26,7 @@ from pydantic import BaseModel, Field, field_validator
from sqlalchemy import select
from app.gateway.authz import require_permission
from app.gateway.deps import get_checkpointer, get_thread_store
from app.gateway.deps import get_checkpointer
from app.gateway.llmwiki_deposit import delete_conversation_deposit_documents
from app.gateway.operation_timing import OperationTiming
from app.gateway.utils import sanitize_log_param
@@ -62,9 +61,7 @@ _RAG_DEBUG_LOGGER_NAME = "rag_citation_debug"
# owner identity through the API surface. Defense-in-depth — the
# row-level invariant is still ``threads_meta.user_id`` populated from
# the auth contextvar; this list closes the metadata-blob echo gap.
_SERVER_RESERVED_METADATA_KEYS: frozenset[str] = frozenset(
{"owner_id", "user_id", "consumer_values_save_called_at"}
)
_SERVER_RESERVED_METADATA_KEYS: frozenset[str] = frozenset({"owner_id", "user_id"})
def _strip_reserved_metadata(metadata: dict[str, Any] | None) -> dict[str, Any]:
@@ -180,7 +177,6 @@ class ThreadCreateRequest(BaseModel):
thread_id: str | None = Field(default=None, description="Optional thread ID (auto-generated if omitted)")
assistant_id: str | None = Field(default=None, description="Associate thread with an assistant")
metadata: dict[str, Any] = Field(default_factory=dict, description="Initial metadata")
initialize_checkpoint: bool = Field(default=False, description="Compatibility option: immediately write an empty checkpoint instead of initializing on first run")
_strip_reserved = field_validator("metadata")(classmethod(lambda cls, v: _strip_reserved_metadata(v)))
@@ -208,19 +204,6 @@ class ThreadStateResponse(BaseModel):
tasks: list[dict[str, Any]] = Field(default_factory=list, description="Interrupted task details")
class ThreadTitleStateResponse(BaseModel):
"""Lightweight title refresh status backed only by thread metadata."""
title: str = ""
title_provisional: bool | None = None
class ConsumerValuesSaveClaimResponse(BaseModel):
"""Whether the caller must backfill Consumer values for this thread."""
should_save: bool
class ThreadMessagesResponse(BaseModel):
"""A page of the latest thread message state.
@@ -877,13 +860,13 @@ async def delete_thread_data(thread_id: str, request: Request) -> ThreadDeleteRe
async def create_thread(body: ThreadCreateRequest, request: Request, response: Response = None) -> ThreadResponse: # type: ignore[assignment]
"""Create a new thread.
Writes thread metadata immediately; the first graph run initializes state.
Legacy callers can explicitly request an immediate empty checkpoint.
Writes a thread_meta record (so the thread appears in /threads/search)
and an empty checkpoint (so state endpoints work immediately).
Idempotent: returns the existing record when ``thread_id`` already exists.
"""
from app.gateway.deps import get_thread_store
checkpointer = get_checkpointer(request) if body.initialize_checkpoint else None
checkpointer = get_checkpointer(request)
thread_store = get_thread_store(request)
thread_id = body.thread_id or str(uuid.uuid4())
now = now_iso()
@@ -896,13 +879,12 @@ async def create_thread(body: ThreadCreateRequest, request: Request, response: R
"thread_create",
thread_id=sanitize_log_param(thread_id),
metadata_store=type(thread_store).__name__,
checkpointer=type(checkpointer).__name__ if checkpointer is not None else "deferred",
checkpointer=type(checkpointer).__name__,
) as timing:
# Idempotency: return existing record when already present
with timing.phase("metadata_lookup"):
existing_record = await thread_store.get(thread_id)
if existing_record is not None:
_mark_primary_only_thread(request, thread_id)
return ThreadResponse(
thread_id=thread_id,
status=existing_record.get("status", "idle"),
@@ -923,12 +905,8 @@ async def create_thread(body: ThreadCreateRequest, request: Request, response: R
logger.exception("Failed to write thread_meta for %s", sanitize_log_param(thread_id))
raise HTTPException(status_code=500, detail="Failed to create thread")
if not body.initialize_checkpoint:
_mark_primary_only_thread(request, thread_id)
return ThreadResponse(thread_id=thread_id, created_at=now, updated_at=now, metadata=body.metadata)
# Explicit compatibility path only. Never schedule this write in the
# background: a late empty checkpoint could replace the first run's head.
# Keep the initial checkpoint: state endpoints must work immediately.
# Its timing includes saver lock/pool waits as well as the DB write.
config = {"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}}
try:
ckpt_metadata = {
@@ -1090,67 +1068,23 @@ async def patch_thread(thread_id: str, body: ThreadPatchRequest, request: Reques
)
@router.post("/{thread_id}/consumer-values-save-claim", response_model=ConsumerValuesSaveClaimResponse)
@require_permission("threads", "write", owner_check=True, require_existing=True)
async def claim_consumer_values_save(
thread_id: str,
request: Request,
) -> ConsumerValuesSaveClaimResponse:
"""Claim the one-time Consumer backfill for a pre-existing conversation.
The timestamp records that the external save request was *attempted*, not
that Consumer accepted it. This matches the non-blocking UI contract: a
Consumer outage must never be surfaced to or block the chat user.
"""
thread_store = get_thread_store(request)
record = await thread_store.get(thread_id)
if record is None:
raise HTTPException(status_code=404, detail=f"Thread {thread_id} not found")
metadata = record.get("metadata") or {}
if metadata.get("consumer_values_save_called_at"):
return ConsumerValuesSaveClaimResponse(should_save=False)
try:
await thread_store.update_metadata(
thread_id,
{"consumer_values_save_called_at": now_iso()},
)
except Exception:
logger.exception("Failed to claim Consumer values save for thread %s", sanitize_log_param(thread_id))
raise HTTPException(status_code=500, detail="Failed to record Consumer values save")
return ConsumerValuesSaveClaimResponse(should_save=True)
@router.get("/{thread_id}", response_model=ThreadResponse)
@require_permission("threads", "read", owner_check=True)
async def get_thread(thread_id: str, request: Request, include_state: bool = False) -> ThreadResponse:
async def get_thread(thread_id: str, request: Request) -> ThreadResponse:
"""Get thread info.
Metadata reads use only the business database. Clients that need channel
values can opt in with include_state, or use /state. Legacy threads without
a metadata row retain the checkpoint fallback after authorization.
Reads metadata from the ThreadMetaStore and derives the accurate
execution status from the checkpointer. Falls back to the checkpointer
alone for threads that pre-date ThreadMetaStore adoption (backward compat).
"""
from app.gateway.deps import get_thread_store
thread_store = get_thread_store(request)
checkpointer = get_checkpointer(request)
# The owner-check decorator already authorized this read, including shared
# task threads. A metadata-only shared thread has no checkpoint fallback.
record: dict | None = await thread_store.get(thread_id, user_id=None)
if record is not None and not include_state:
return ThreadResponse(
thread_id=thread_id,
status="busy" if record.get("status") == "running" else record.get("status", "idle"),
created_at=coerce_iso(record.get("created_at", "")),
updated_at=coerce_iso(record.get("updated_at", "")),
metadata=record.get("metadata", {}),
)
record: dict | None = await thread_store.get(thread_id)
# Derive accurate status from the checkpointer
checkpointer = get_checkpointer(request)
config = {"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}}
try:
checkpoint_tuple = await checkpointer.aget_tuple(config)
@@ -1187,91 +1121,61 @@ async def get_thread(thread_id: str, request: Request, include_state: bool = Fal
created_at=coerce_iso(record.get("created_at", "")),
updated_at=coerce_iso(record.get("updated_at", "")),
metadata=record.get("metadata", {}),
values=serialize_channel_values(channel_values, slim_display=True),
values=serialize_channel_values(channel_values),
)
# ---------------------------------------------------------------------------
@router.get("/{thread_id}/title-state", response_model=ThreadTitleStateResponse)
@require_permission("threads", "read", owner_check=True)
async def get_thread_title_state(thread_id: str, request: Request) -> ThreadTitleStateResponse:
"""Read only title metadata; do not load/serialize a checkpoint or messages."""
record = await get_thread_store(request).get(thread_id)
if record is None:
raise HTTPException(status_code=404, detail=f"Thread {thread_id} not found")
metadata = record.get("metadata") or {}
provisional = metadata.get("_deerflow_title_provisional")
return ThreadTitleStateResponse(
title=str(record.get("display_name") or ""),
title_provisional=provisional if isinstance(provisional, bool) else None,
)
@router.get("/{thread_id}/state", response_model=ThreadStateResponse)
@require_permission("threads", "read", owner_check=True)
async def get_thread_state(thread_id: str, request: Request, response: Response = None) -> ThreadStateResponse: # type: ignore[assignment]
async def get_thread_state(thread_id: str, request: Request) -> ThreadStateResponse:
"""Get the latest state snapshot for a thread.
Channel values are serialized to ensure LangChain message objects
are converted to JSON-safe dicts. Oversized tool/file bodies are
truncated for this hydrate snapshot; the checkpoint itself is unchanged.
are converted to JSON-safe dicts.
"""
checkpointer = get_checkpointer(request)
config = {"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}}
with OperationTiming(
response,
logger,
"thread_state",
thread_id=sanitize_log_param(thread_id),
checkpointer=type(checkpointer).__name__,
) as timing:
try:
with timing.phase("checkpoint_load"):
checkpoint_tuple = await checkpointer.aget_tuple(config)
except Exception:
logger.exception("Failed to get state for thread %s", sanitize_log_param(thread_id))
raise HTTPException(status_code=500, detail="Failed to get thread state")
try:
checkpoint_tuple = await checkpointer.aget_tuple(config)
except Exception:
logger.exception("Failed to get state for thread %s", sanitize_log_param(thread_id))
raise HTTPException(status_code=500, detail="Failed to get thread state")
if checkpoint_tuple is None:
from app.gateway.deps import get_thread_store
if checkpoint_tuple is None:
raise HTTPException(status_code=404, detail=f"Thread {thread_id} not found")
record = await get_thread_store(request).get(thread_id, user_id=None)
if record is not None:
return ThreadStateResponse(metadata=record.get("metadata", {}))
raise HTTPException(status_code=404, detail=f"Thread {thread_id} not found")
checkpoint = getattr(checkpoint_tuple, "checkpoint", {}) or {}
metadata = getattr(checkpoint_tuple, "metadata", {}) or {}
checkpoint_id = None
ckpt_config = getattr(checkpoint_tuple, "config", {})
if ckpt_config:
checkpoint_id = ckpt_config.get("configurable", {}).get("checkpoint_id")
checkpoint = getattr(checkpoint_tuple, "checkpoint", {}) or {}
metadata = getattr(checkpoint_tuple, "metadata", {}) or {}
checkpoint_id = None
ckpt_config = getattr(checkpoint_tuple, "config", {})
if ckpt_config:
checkpoint_id = ckpt_config.get("configurable", {}).get("checkpoint_id")
channel_values = checkpoint.get("channel_values", {})
channel_values = checkpoint.get("channel_values", {})
parent_config = getattr(checkpoint_tuple, "parent_config", None)
parent_checkpoint_id = None
if parent_config:
parent_checkpoint_id = parent_config.get("configurable", {}).get("checkpoint_id")
parent_config = getattr(checkpoint_tuple, "parent_config", None)
parent_checkpoint_id = None
if parent_config:
parent_checkpoint_id = parent_config.get("configurable", {}).get("checkpoint_id")
tasks_raw = getattr(checkpoint_tuple, "tasks", []) or []
next_tasks = [t.name for t in tasks_raw if hasattr(t, "name")]
tasks = [{"id": getattr(t, "id", ""), "name": getattr(t, "name", "")} for t in tasks_raw]
tasks_raw = getattr(checkpoint_tuple, "tasks", []) or []
next_tasks = [t.name for t in tasks_raw if hasattr(t, "name")]
tasks = [{"id": getattr(t, "id", ""), "name": getattr(t, "name", "")} for t in tasks_raw]
values = serialize_channel_values(channel_values)
with timing.phase("serialize"):
values = serialize_channel_values(channel_values, slim_display=True)
return ThreadStateResponse(
values=values,
next=next_tasks,
metadata=metadata,
checkpoint={"id": checkpoint_id, "ts": coerce_iso(metadata.get("created_at", ""))},
checkpoint_id=checkpoint_id,
parent_checkpoint_id=parent_checkpoint_id,
created_at=coerce_iso(metadata.get("created_at", "")),
tasks=tasks,
)
return ThreadStateResponse(
values=values,
next=next_tasks,
metadata=metadata,
checkpoint={"id": checkpoint_id, "ts": coerce_iso(metadata.get("created_at", ""))},
checkpoint_id=checkpoint_id,
parent_checkpoint_id=parent_checkpoint_id,
created_at=coerce_iso(metadata.get("created_at", "")),
tasks=tasks,
)
@router.get("/{thread_id}/reference-batches", response_model=ReferenceBatchesResponse)
@@ -2008,50 +1912,9 @@ async def edit_thread_message(
}
def _mark_primary_only_thread(request: Request, thread_id: str) -> None:
"""Tell a dual-read checkpointer not to scan legacy SQLite for this thread."""
saver = getattr(request.app.state, "checkpointer", None)
remember = getattr(saver, "remember_primary_thread", None)
if callable(remember):
remember(thread_id)
def _history_entry_from_tuple(checkpoint_tuple: Any, *, include_values: bool) -> HistoryEntry:
"""Build one history card. Only the latest head carries serialized values."""
ckpt_config = getattr(checkpoint_tuple, "config", {})
parent_config = getattr(checkpoint_tuple, "parent_config", None)
metadata = getattr(checkpoint_tuple, "metadata", {}) or {}
checkpoint = getattr(checkpoint_tuple, "checkpoint", {}) or {}
checkpoint_id = ckpt_config.get("configurable", {}).get("checkpoint_id", "")
parent_id = parent_config.get("configurable", {}).get("checkpoint_id") if parent_config else None
channel_values = checkpoint.get("channel_values", {})
values: dict[str, Any] = serialize_channel_values(channel_values, slim_display=True) if include_values else {}
if title := channel_values.get("title"):
values["title"] = title
if thread_data := channel_values.get("thread_data"):
values["thread_data"] = thread_data
tasks_raw = getattr(checkpoint_tuple, "tasks", []) or []
user_meta = {k: v for k, v in metadata.items() if k not in ("created_at", "updated_at", "step", "source", "writes", "parents")}
if "step" in metadata:
user_meta["step"] = metadata["step"]
return HistoryEntry(
checkpoint_id=checkpoint_id,
parent_checkpoint_id=parent_id,
metadata=user_meta,
values=values,
created_at=coerce_iso(metadata.get("created_at", "")),
next=[t.name for t in tasks_raw if hasattr(t, "name")],
)
@router.post("/{thread_id}/history", response_model=list[HistoryEntry])
@require_permission("threads", "read", owner_check=True)
async def get_thread_history(
thread_id: str,
body: ThreadHistoryRequest,
request: Request,
response: Response = None, # type: ignore[assignment]
) -> list[HistoryEntry]:
async def get_thread_history(thread_id: str, body: ThreadHistoryRequest, request: Request) -> list[HistoryEntry]:
"""Get checkpoint history for a thread.
Messages are read from the checkpointer's channel values (the
@@ -2059,37 +1922,65 @@ async def get_thread_history(
:func:`~deerflow.runtime.serialization.serialize_channel_values`.
Only the latest (first) checkpoint carries the ``messages`` key to
avoid duplicating them across every entry.
The chat UI always asks for ``limit=1``. That path uses ``aget_tuple``
instead of walking the checkpoint iterator.
"""
checkpointer = get_checkpointer(request)
config: dict[str, Any] = {"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}}
config: dict[str, Any] = {"configurable": {"thread_id": thread_id}}
if body.before:
config["configurable"]["checkpoint_id"] = body.before
with OperationTiming(
response,
logger,
"thread_history",
thread_id=sanitize_log_param(thread_id),
checkpointer=type(checkpointer).__name__,
limit=str(body.limit),
) as timing:
try:
with timing.phase("checkpoint_load"):
if body.limit == 1 and not body.before:
latest = await checkpointer.aget_tuple(config)
checkpoints = [latest] if latest is not None else []
else:
async with aclosing(checkpointer.alist(config, limit=body.limit)) as history:
checkpoints = [checkpoint async for checkpoint in history]
with timing.phase("serialize"):
return [
_history_entry_from_tuple(checkpoint_tuple, include_values=index == 0)
for index, checkpoint_tuple in enumerate(checkpoints)
]
except Exception:
logger.exception("Failed to get history for thread %s", sanitize_log_param(thread_id))
raise HTTPException(status_code=500, detail="Failed to get thread history")
entries: list[HistoryEntry] = []
is_latest_checkpoint = True
try:
async for checkpoint_tuple in checkpointer.alist(config, limit=body.limit):
ckpt_config = getattr(checkpoint_tuple, "config", {})
parent_config = getattr(checkpoint_tuple, "parent_config", None)
metadata = getattr(checkpoint_tuple, "metadata", {}) or {}
checkpoint = getattr(checkpoint_tuple, "checkpoint", {}) or {}
checkpoint_id = ckpt_config.get("configurable", {}).get("checkpoint_id", "")
parent_id = None
if parent_config:
parent_id = parent_config.get("configurable", {}).get("checkpoint_id")
channel_values = checkpoint.get("channel_values", {})
# Build values from checkpoint channel_values
values: dict[str, Any] = {}
if title := channel_values.get("title"):
values["title"] = title
if thread_data := channel_values.get("thread_data"):
values["thread_data"] = thread_data
# Attach messages only to the latest checkpoint entry.
if is_latest_checkpoint:
messages = channel_values.get("messages")
if messages:
values["messages"] = serialize_channel_values({"messages": messages}).get("messages", [])
is_latest_checkpoint = False
# Derive next tasks
tasks_raw = getattr(checkpoint_tuple, "tasks", []) or []
next_tasks = [t.name for t in tasks_raw if hasattr(t, "name")]
# Strip LangGraph internal keys from metadata
user_meta = {k: v for k, v in metadata.items() if k not in ("created_at", "updated_at", "step", "source", "writes", "parents")}
# Keep step for ordering context
if "step" in metadata:
user_meta["step"] = metadata["step"]
entries.append(
HistoryEntry(
checkpoint_id=checkpoint_id,
parent_checkpoint_id=parent_id,
metadata=user_meta,
values=values,
created_at=coerce_iso(metadata.get("created_at", "")),
next=next_tasks,
)
)
except Exception:
logger.exception("Failed to get history for thread %s", sanitize_log_param(thread_id))
raise HTTPException(status_code=500, detail="Failed to get thread history")
return entries