sync update from LAN repo (source master @ ca48c4ec)

This commit is contained in:
gozeros committed 2026-11-05 10:53:54 +08:00
1 parent 8bc8b0e082
commit ea9dcff9c0
873 files changed
+107866 -14011

No files matched your search

@@ -17,6 +17,7 @@ import json
import logging
import os
import uuid
from contextlib import aclosing
from pathlib import Path
from typing import Any
@@ -26,7 +27,7 @@ from pydantic import BaseModel, Field, field_validator
from sqlalchemy import select
from app.gateway.authz import require_permission
from app.gateway.deps import get_checkpointer
from app.gateway.deps import get_checkpointer, get_thread_store
from app.gateway.llmwiki_deposit import delete_conversation_deposit_documents
from app.gateway.operation_timing import OperationTiming
from app.gateway.utils import sanitize_log_param
@@ -61,7 +62,9 @@ _RAG_DEBUG_LOGGER_NAME = "rag_citation_debug"
# owner identity through the API surface. Defense-in-depth — the
# row-level invariant is still ``threads_meta.user_id`` populated from
# the auth contextvar; this list closes the metadata-blob echo gap.
_SERVER_RESERVED_METADATA_KEYS: frozenset[str] = frozenset({"owner_id", "user_id"})
_SERVER_RESERVED_METADATA_KEYS: frozenset[str] = frozenset(
{"owner_id", "user_id", "consumer_values_save_called_at"}
)
def _strip_reserved_metadata(metadata: dict[str, Any] | None) -> dict[str, Any]:
@@ -177,6 +180,7 @@ class ThreadCreateRequest(BaseModel):
thread_id: str | None = Field(default=None, description="Optional thread ID (auto-generated if omitted)")
assistant_id: str | None = Field(default=None, description="Associate thread with an assistant")
metadata: dict[str, Any] = Field(default_factory=dict, description="Initial metadata")
initialize_checkpoint: bool = Field(default=False, description="Compatibility option: immediately write an empty checkpoint instead of initializing on first run")
_strip_reserved = field_validator("metadata")(classmethod(lambda cls, v: _strip_reserved_metadata(v)))
@@ -204,6 +208,19 @@ class ThreadStateResponse(BaseModel):
tasks: list[dict[str, Any]] = Field(default_factory=list, description="Interrupted task details")
class ThreadTitleStateResponse(BaseModel):
"""Lightweight title refresh status backed only by thread metadata."""
title: str = ""
title_provisional: bool | None = None
class ConsumerValuesSaveClaimResponse(BaseModel):
"""Whether the caller must backfill Consumer values for this thread."""
should_save: bool
class ThreadMessagesResponse(BaseModel):
"""A page of the latest thread message state.
@@ -860,13 +877,13 @@ async def delete_thread_data(thread_id: str, request: Request) -> ThreadDeleteRe
async def create_thread(body: ThreadCreateRequest, request: Request, response: Response = None) -> ThreadResponse: # type: ignore[assignment]
"""Create a new thread.
Writes a thread_meta record (so the thread appears in /threads/search)
and an empty checkpoint (so state endpoints work immediately).
Writes thread metadata immediately; the first graph run initializes state.
Legacy callers can explicitly request an immediate empty checkpoint.
Idempotent: returns the existing record when ``thread_id`` already exists.
"""
from app.gateway.deps import get_thread_store
checkpointer = get_checkpointer(request)
checkpointer = get_checkpointer(request) if body.initialize_checkpoint else None
thread_store = get_thread_store(request)
thread_id = body.thread_id or str(uuid.uuid4())
now = now_iso()
@@ -879,12 +896,13 @@ async def create_thread(body: ThreadCreateRequest, request: Request, response: R
"thread_create",
thread_id=sanitize_log_param(thread_id),
metadata_store=type(thread_store).__name__,
checkpointer=type(checkpointer).__name__,
checkpointer=type(checkpointer).__name__ if checkpointer is not None else "deferred",
) as timing:
# Idempotency: return existing record when already present
with timing.phase("metadata_lookup"):
existing_record = await thread_store.get(thread_id)
if existing_record is not None:
_mark_primary_only_thread(request, thread_id)
return ThreadResponse(
thread_id=thread_id,
status=existing_record.get("status", "idle"),
@@ -905,8 +923,12 @@ async def create_thread(body: ThreadCreateRequest, request: Request, response: R
logger.exception("Failed to write thread_meta for %s", sanitize_log_param(thread_id))
raise HTTPException(status_code=500, detail="Failed to create thread")
# Keep the initial checkpoint: state endpoints must work immediately.
# Its timing includes saver lock/pool waits as well as the DB write.
if not body.initialize_checkpoint:
_mark_primary_only_thread(request, thread_id)
return ThreadResponse(thread_id=thread_id, created_at=now, updated_at=now, metadata=body.metadata)
# Explicit compatibility path only. Never schedule this write in the
# background: a late empty checkpoint could replace the first run's head.
config = {"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}}
try:
ckpt_metadata = {
@@ -1068,23 +1090,67 @@ async def patch_thread(thread_id: str, body: ThreadPatchRequest, request: Reques
)
@router.post("/{thread_id}/consumer-values-save-claim", response_model=ConsumerValuesSaveClaimResponse)
@require_permission("threads", "write", owner_check=True, require_existing=True)
async def claim_consumer_values_save(
thread_id: str,
request: Request,
) -> ConsumerValuesSaveClaimResponse:
"""Claim the one-time Consumer backfill for a pre-existing conversation.
The timestamp records that the external save request was *attempted*, not
that Consumer accepted it. This matches the non-blocking UI contract: a
Consumer outage must never be surfaced to or block the chat user.
"""
thread_store = get_thread_store(request)
record = await thread_store.get(thread_id)
if record is None:
raise HTTPException(status_code=404, detail=f"Thread {thread_id} not found")
metadata = record.get("metadata") or {}
if metadata.get("consumer_values_save_called_at"):
return ConsumerValuesSaveClaimResponse(should_save=False)
try:
await thread_store.update_metadata(
thread_id,
{"consumer_values_save_called_at": now_iso()},
)
except Exception:
logger.exception("Failed to claim Consumer values save for thread %s", sanitize_log_param(thread_id))
raise HTTPException(status_code=500, detail="Failed to record Consumer values save")
return ConsumerValuesSaveClaimResponse(should_save=True)
@router.get("/{thread_id}", response_model=ThreadResponse)
@require_permission("threads", "read", owner_check=True)
async def get_thread(thread_id: str, request: Request) -> ThreadResponse:
async def get_thread(thread_id: str, request: Request, include_state: bool = False) -> ThreadResponse:
"""Get thread info.
Reads metadata from the ThreadMetaStore and derives the accurate
execution status from the checkpointer. Falls back to the checkpointer
alone for threads that pre-date ThreadMetaStore adoption (backward compat).
Metadata reads use only the business database. Clients that need channel
values can opt in with include_state, or use /state. Legacy threads without
a metadata row retain the checkpoint fallback after authorization.
"""
from app.gateway.deps import get_thread_store
thread_store = get_thread_store(request)
checkpointer = get_checkpointer(request)
record: dict | None = await thread_store.get(thread_id)
# The owner-check decorator already authorized this read, including shared
# task threads. A metadata-only shared thread has no checkpoint fallback.
record: dict | None = await thread_store.get(thread_id, user_id=None)
if record is not None and not include_state:
return ThreadResponse(
thread_id=thread_id,
status="busy" if record.get("status") == "running" else record.get("status", "idle"),
created_at=coerce_iso(record.get("created_at", "")),
updated_at=coerce_iso(record.get("updated_at", "")),
metadata=record.get("metadata", {}),
)
# Derive accurate status from the checkpointer
checkpointer = get_checkpointer(request)
config = {"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}}
try:
checkpoint_tuple = await checkpointer.aget_tuple(config)
@@ -1121,61 +1187,91 @@ async def get_thread(thread_id: str, request: Request) -> ThreadResponse:
created_at=coerce_iso(record.get("created_at", "")),
updated_at=coerce_iso(record.get("updated_at", "")),
metadata=record.get("metadata", {}),
values=serialize_channel_values(channel_values),
values=serialize_channel_values(channel_values, slim_display=True),
)
# ---------------------------------------------------------------------------
@router.get("/{thread_id}/title-state", response_model=ThreadTitleStateResponse)
@require_permission("threads", "read", owner_check=True)
async def get_thread_title_state(thread_id: str, request: Request) -> ThreadTitleStateResponse:
"""Read only title metadata; do not load/serialize a checkpoint or messages."""
record = await get_thread_store(request).get(thread_id)
if record is None:
raise HTTPException(status_code=404, detail=f"Thread {thread_id} not found")
metadata = record.get("metadata") or {}
provisional = metadata.get("_deerflow_title_provisional")
return ThreadTitleStateResponse(
title=str(record.get("display_name") or ""),
title_provisional=provisional if isinstance(provisional, bool) else None,
)
@router.get("/{thread_id}/state", response_model=ThreadStateResponse)
@require_permission("threads", "read", owner_check=True)
async def get_thread_state(thread_id: str, request: Request) -> ThreadStateResponse:
async def get_thread_state(thread_id: str, request: Request, response: Response = None) -> ThreadStateResponse: # type: ignore[assignment]
"""Get the latest state snapshot for a thread.
Channel values are serialized to ensure LangChain message objects
are converted to JSON-safe dicts.
are converted to JSON-safe dicts. Oversized tool/file bodies are
truncated for this hydrate snapshot; the checkpoint itself is unchanged.
"""
checkpointer = get_checkpointer(request)
config = {"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}}
try:
checkpoint_tuple = await checkpointer.aget_tuple(config)
except Exception:
logger.exception("Failed to get state for thread %s", sanitize_log_param(thread_id))
raise HTTPException(status_code=500, detail="Failed to get thread state")
with OperationTiming(
response,
logger,
"thread_state",
thread_id=sanitize_log_param(thread_id),
checkpointer=type(checkpointer).__name__,
) as timing:
try:
with timing.phase("checkpoint_load"):
checkpoint_tuple = await checkpointer.aget_tuple(config)
except Exception:
logger.exception("Failed to get state for thread %s", sanitize_log_param(thread_id))
raise HTTPException(status_code=500, detail="Failed to get thread state")
if checkpoint_tuple is None:
raise HTTPException(status_code=404, detail=f"Thread {thread_id} not found")
if checkpoint_tuple is None:
from app.gateway.deps import get_thread_store
checkpoint = getattr(checkpoint_tuple, "checkpoint", {}) or {}
metadata = getattr(checkpoint_tuple, "metadata", {}) or {}
checkpoint_id = None
ckpt_config = getattr(checkpoint_tuple, "config", {})
if ckpt_config:
checkpoint_id = ckpt_config.get("configurable", {}).get("checkpoint_id")
record = await get_thread_store(request).get(thread_id, user_id=None)
if record is not None:
return ThreadStateResponse(metadata=record.get("metadata", {}))
raise HTTPException(status_code=404, detail=f"Thread {thread_id} not found")
channel_values = checkpoint.get("channel_values", {})
checkpoint = getattr(checkpoint_tuple, "checkpoint", {}) or {}
metadata = getattr(checkpoint_tuple, "metadata", {}) or {}
checkpoint_id = None
ckpt_config = getattr(checkpoint_tuple, "config", {})
if ckpt_config:
checkpoint_id = ckpt_config.get("configurable", {}).get("checkpoint_id")
parent_config = getattr(checkpoint_tuple, "parent_config", None)
parent_checkpoint_id = None
if parent_config:
parent_checkpoint_id = parent_config.get("configurable", {}).get("checkpoint_id")
channel_values = checkpoint.get("channel_values", {})
tasks_raw = getattr(checkpoint_tuple, "tasks", []) or []
next_tasks = [t.name for t in tasks_raw if hasattr(t, "name")]
tasks = [{"id": getattr(t, "id", ""), "name": getattr(t, "name", "")} for t in tasks_raw]
parent_config = getattr(checkpoint_tuple, "parent_config", None)
parent_checkpoint_id = None
if parent_config:
parent_checkpoint_id = parent_config.get("configurable", {}).get("checkpoint_id")
values = serialize_channel_values(channel_values)
tasks_raw = getattr(checkpoint_tuple, "tasks", []) or []
next_tasks = [t.name for t in tasks_raw if hasattr(t, "name")]
tasks = [{"id": getattr(t, "id", ""), "name": getattr(t, "name", "")} for t in tasks_raw]
return ThreadStateResponse(
values=values,
next=next_tasks,
metadata=metadata,
checkpoint={"id": checkpoint_id, "ts": coerce_iso(metadata.get("created_at", ""))},
checkpoint_id=checkpoint_id,
parent_checkpoint_id=parent_checkpoint_id,
created_at=coerce_iso(metadata.get("created_at", "")),
tasks=tasks,
)
with timing.phase("serialize"):
values = serialize_channel_values(channel_values, slim_display=True)
return ThreadStateResponse(
values=values,
next=next_tasks,
metadata=metadata,
checkpoint={"id": checkpoint_id, "ts": coerce_iso(metadata.get("created_at", ""))},
checkpoint_id=checkpoint_id,
parent_checkpoint_id=parent_checkpoint_id,
created_at=coerce_iso(metadata.get("created_at", "")),
tasks=tasks,
)
@router.get("/{thread_id}/reference-batches", response_model=ReferenceBatchesResponse)
@@ -1912,9 +2008,50 @@ async def edit_thread_message(
}
def _mark_primary_only_thread(request: Request, thread_id: str) -> None:
"""Tell a dual-read checkpointer not to scan legacy SQLite for this thread."""
saver = getattr(request.app.state, "checkpointer", None)
remember = getattr(saver, "remember_primary_thread", None)
if callable(remember):
remember(thread_id)
def _history_entry_from_tuple(checkpoint_tuple: Any, *, include_values: bool) -> HistoryEntry:
"""Build one history card. Only the latest head carries serialized values."""
ckpt_config = getattr(checkpoint_tuple, "config", {})
parent_config = getattr(checkpoint_tuple, "parent_config", None)
metadata = getattr(checkpoint_tuple, "metadata", {}) or {}
checkpoint = getattr(checkpoint_tuple, "checkpoint", {}) or {}
checkpoint_id = ckpt_config.get("configurable", {}).get("checkpoint_id", "")
parent_id = parent_config.get("configurable", {}).get("checkpoint_id") if parent_config else None
channel_values = checkpoint.get("channel_values", {})
values: dict[str, Any] = serialize_channel_values(channel_values, slim_display=True) if include_values else {}
if title := channel_values.get("title"):
values["title"] = title
if thread_data := channel_values.get("thread_data"):
values["thread_data"] = thread_data
tasks_raw = getattr(checkpoint_tuple, "tasks", []) or []
user_meta = {k: v for k, v in metadata.items() if k not in ("created_at", "updated_at", "step", "source", "writes", "parents")}
if "step" in metadata:
user_meta["step"] = metadata["step"]
return HistoryEntry(
checkpoint_id=checkpoint_id,
parent_checkpoint_id=parent_id,
metadata=user_meta,
values=values,
created_at=coerce_iso(metadata.get("created_at", "")),
next=[t.name for t in tasks_raw if hasattr(t, "name")],
)
@router.post("/{thread_id}/history", response_model=list[HistoryEntry])
@require_permission("threads", "read", owner_check=True)
async def get_thread_history(thread_id: str, body: ThreadHistoryRequest, request: Request) -> list[HistoryEntry]:
async def get_thread_history(
thread_id: str,
body: ThreadHistoryRequest,
request: Request,
response: Response = None, # type: ignore[assignment]
) -> list[HistoryEntry]:
"""Get checkpoint history for a thread.
Messages are read from the checkpointer's channel values (the
@@ -1922,65 +2059,37 @@ async def get_thread_history(thread_id: str, body: ThreadHistoryRequest, request
:func:`~deerflow.runtime.serialization.serialize_channel_values`.
Only the latest (first) checkpoint carries the ``messages`` key to
avoid duplicating them across every entry.
The chat UI always asks for ``limit=1``. That path uses ``aget_tuple``
instead of walking the checkpoint iterator.
"""
checkpointer = get_checkpointer(request)
config: dict[str, Any] = {"configurable": {"thread_id": thread_id}}
config: dict[str, Any] = {"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}}
if body.before:
config["configurable"]["checkpoint_id"] = body.before
entries: list[HistoryEntry] = []
is_latest_checkpoint = True
try:
async for checkpoint_tuple in checkpointer.alist(config, limit=body.limit):
ckpt_config = getattr(checkpoint_tuple, "config", {})
parent_config = getattr(checkpoint_tuple, "parent_config", None)
metadata = getattr(checkpoint_tuple, "metadata", {}) or {}
checkpoint = getattr(checkpoint_tuple, "checkpoint", {}) or {}
checkpoint_id = ckpt_config.get("configurable", {}).get("checkpoint_id", "")
parent_id = None
if parent_config:
parent_id = parent_config.get("configurable", {}).get("checkpoint_id")
channel_values = checkpoint.get("channel_values", {})
# Build values from checkpoint channel_values
values: dict[str, Any] = {}
if title := channel_values.get("title"):
values["title"] = title
if thread_data := channel_values.get("thread_data"):
values["thread_data"] = thread_data
# Attach messages only to the latest checkpoint entry.
if is_latest_checkpoint:
messages = channel_values.get("messages")
if messages:
values["messages"] = serialize_channel_values({"messages": messages}).get("messages", [])
is_latest_checkpoint = False
# Derive next tasks
tasks_raw = getattr(checkpoint_tuple, "tasks", []) or []
next_tasks = [t.name for t in tasks_raw if hasattr(t, "name")]
# Strip LangGraph internal keys from metadata
user_meta = {k: v for k, v in metadata.items() if k not in ("created_at", "updated_at", "step", "source", "writes", "parents")}
# Keep step for ordering context
if "step" in metadata:
user_meta["step"] = metadata["step"]
entries.append(
HistoryEntry(
checkpoint_id=checkpoint_id,
parent_checkpoint_id=parent_id,
metadata=user_meta,
values=values,
created_at=coerce_iso(metadata.get("created_at", "")),
next=next_tasks,
)
)
except Exception:
logger.exception("Failed to get history for thread %s", sanitize_log_param(thread_id))
raise HTTPException(status_code=500, detail="Failed to get thread history")
return entries
with OperationTiming(
response,
logger,
"thread_history",
thread_id=sanitize_log_param(thread_id),
checkpointer=type(checkpointer).__name__,
limit=str(body.limit),
) as timing:
try:
with timing.phase("checkpoint_load"):
if body.limit == 1 and not body.before:
latest = await checkpointer.aget_tuple(config)
checkpoints = [latest] if latest is not None else []
else:
async with aclosing(checkpointer.alist(config, limit=body.limit)) as history:
checkpoints = [checkpoint async for checkpoint in history]
with timing.phase("serialize"):
return [
_history_entry_from_tuple(checkpoint_tuple, include_values=index == 0)
for index, checkpoint_tuple in enumerate(checkpoints)
]
except Exception:
logger.exception("Failed to get history for thread %s", sanitize_log_param(thread_id))
raise HTTPException(status_code=500, detail="Failed to get thread history")