sync: 鍚屾灞€鍩熺綉鏈€鏂颁唬鐮?(source master @ b69568e4)

This commit is contained in:
gozeros committed 2026-10-21 14:51:31 +08:00
1 parent 4f7f07504a
commit 8bc8b0e082
1651 files changed
+278599 -11416

No files matched your search

@@ -30,19 +30,49 @@ EXCLUDED_SYSTEM_THREAD_TYPES = frozenset(
"deep_research",
"writing_setup",
"report_structure_setup",
"task-analysis",
"task-analysis-qa",
"task_deeplink",
}
)
# 这些系统线程 id 只存在于编排内部,不进「最近的对话」;loopback 派活必须能跨
# 「点击触发的 HTTP 用户」和「绑定 created_by 签发的 JWT」读写,否则会 404 not accessible。
SHARED_ACCESS_THREAD_TYPES = frozenset(
{
"roundtable",
"task-analysis",
"task-analysis-qa",
}
)
def is_excluded_system_thread(metadata: dict[str, Any] | None) -> bool:
"""Return True when a thread should be hidden from user conversation lists."""
"""Return True when a thread should be hidden from user conversation lists.
Task-analysis seat runs created before ``thread_type`` was stamped still
carry ``task_analysis_seat``. Those rows stay in the execution record and
out of recent chats.
"""
md = metadata or {}
if md.get("system"):
return True
seat = md.get("task_analysis_seat")
if isinstance(seat, str) and seat.strip():
return True
thread_type = md.get("thread_type")
return isinstance(thread_type, str) and thread_type in EXCLUDED_SYSTEM_THREAD_TYPES
def is_shared_access_thread(metadata: dict[str, Any] | None) -> bool:
"""Return True when any authenticated caller may read/write this thread."""
md = metadata or {}
if md.get("taskId"):
return True
thread_type = md.get("thread_type")
return isinstance(thread_type, str) and thread_type in SHARED_ACCESS_THREAD_TYPES
class ThreadMetaStore(abc.ABC):
@abc.abstractmethod
async def create(
@@ -11,7 +11,15 @@ from typing import Any
from langgraph.store.base import BaseStore
from deerflow.persistence.thread_meta.base import ThreadMetaStore, is_excluded_system_thread
from deerflow.persistence.task_deeplink_threads.memory import (
delete_task_deeplink_index,
sync_task_deeplink_index,
)
from deerflow.persistence.thread_meta.base import (
ThreadMetaStore,
is_excluded_system_thread,
is_shared_access_thread,
)
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
from deerflow.utils.time import coerce_iso, now_iso
@@ -64,6 +72,7 @@ class MemoryThreadMetaStore(ThreadMetaStore):
"updated_at": now,
}
await self._store.aput(THREADS_NS, thread_id, record)
await sync_task_deeplink_index(self._store, record)
return record
async def get(self, thread_id: str, *, user_id: str | None | _AutoSentinel = AUTO) -> dict | None:
@@ -183,8 +192,8 @@ class MemoryThreadMetaStore(ThreadMetaStore):
item = await self._store.aget(THREADS_NS, thread_id)
if item is None:
return not require_existing
# 任务深链对话(metadata.taskId)全局共享、不区分用户:任何登录用户都可读写/续聊。
if (item.value.get("metadata") or {}).get("taskId"):
# 任务深链 / 圆桌 / 任务分析系统线程:全局共享(与 SQL store 同口径)。
if is_shared_access_thread(item.value.get("metadata") or {}):
return True
record_user_id = item.value.get("user_id")
if record_user_id is None:
@@ -224,11 +233,13 @@ class MemoryThreadMetaStore(ThreadMetaStore):
record["metadata"] = merged
record["updated_at"] = now_iso()
await self._store.aput(THREADS_NS, thread_id, record)
await sync_task_deeplink_index(self._store, record)
async def delete(self, thread_id: str, *, user_id: str | None | _AutoSentinel = AUTO) -> None:
record = await self._get_owned_record(thread_id, user_id, "MemoryThreadMetaStore.delete")
if record is None:
return
await delete_task_deeplink_index(self._store, thread_id)
await self._store.adelete(THREADS_NS, thread_id)
@staticmethod
@@ -16,12 +16,27 @@ class ThreadMetaRow(Base):
__table_args__ = (
Index("ix_threads_meta_created_user", "created_at", "user_id"),
Index("ix_threads_meta_user_created", "user_id", "created_at"),
# Task deep-link history is globally shared and therefore cannot use
# the normal ``user_id`` scope. Keep its high-cardinality lookup
# fields out of ``metadata_json`` so task entry never has to scan and
# Python-filter every thread in the installation.
Index("ix_threads_meta_task_agent_updated", "task_id", "agent_id", "updated_at", "thread_id"),
Index("ix_threads_meta_task_action_agent_updated", "task_id", "action_id", "agent_id", "updated_at", "thread_id"),
# The ordinary recent-chat path has an owner filter and the same
# updated-at ordering. This prevents a full table sort there too.
Index("ix_threads_meta_user_updated", "user_id", "updated_at", "thread_id"),
)
thread_id: Mapped[str] = mapped_column(String(64), primary_key=True)
assistant_id: Mapped[str | None] = mapped_column(String(128), index=True)
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
display_name: Mapped[str | None] = mapped_column(String(256))
assistant_id: Mapped[str | None] = mapped_column(String(128), index=True)
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
# Indexed projections of the corresponding metadata values. The JSON
# blob remains authoritative and supports arbitrary metadata filters;
# these columns exist solely for the task-workspace hot path.
task_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
agent_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
action_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
display_name: Mapped[str | None] = mapped_column(String(256))
status: Mapped[str] = mapped_column(String(20), default="idle")
metadata_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
@@ -8,12 +8,69 @@ from typing import Any
from sqlalchemy import func, select, update
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.thread_meta.base import ThreadMetaStore, is_excluded_system_thread
from deerflow.persistence.task_deeplink_threads.sql import (
delete_task_deeplink_index,
sync_task_deeplink_index,
)
from deerflow.persistence.thread_meta.base import ThreadMetaStore, is_excluded_system_thread, is_shared_access_thread
from deerflow.persistence.thread_meta.model import ThreadMetaRow
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
class ThreadMetaRepository(ThreadMetaStore):
_INDEXED_METADATA_COLUMNS = {
"taskId": ThreadMetaRow.task_id,
"agent_id": ThreadMetaRow.agent_id,
"actionId": ThreadMetaRow.action_id,
}
_INDEXED_METADATA_ATTRIBUTES = {
"taskId": "task_id",
"agent_id": "agent_id",
"actionId": "action_id",
}
def _indexed_metadata_values(metadata: dict[str, Any] | None) -> dict[str, str]:
"""Return metadata predicates that have a maintained SQL projection.
Task deep-link callers use string ids. Restrict the SQL shortcut to
strings so legacy callers that intentionally store another JSON type keep
the exact Python-side equality semantics of the generic metadata filter.
"""
return {
key: value
for key, value in (metadata or {}).items()
if key in _INDEXED_METADATA_COLUMNS and isinstance(value, str)
}
def _indexed_thread_metadata(metadata: dict[str, Any] | None) -> dict[str, str | None]:
"""Build ORM column values from authoritative thread metadata.
Always include every projection so a metadata update that changes a value
from a string to another JSON type cannot leave a stale indexed value.
"""
values = _indexed_metadata_values(metadata)
return {
attribute: values.get(metadata_key)
for metadata_key, attribute in _INDEXED_METADATA_ATTRIBUTES.items()
}
def _apply_indexed_metadata_filters(stmt: Any, metadata: dict[str, Any] | None) -> tuple[Any, set[str]]:
"""Apply known metadata predicates in SQL and return their keys.
The caller still verifies the full metadata object in Python whenever a
generic filter or system-thread exclusion is involved. This keeps the
public API's exact-match contract while reducing that verification set to
rows for the requested task/agent/action instead of the entire table.
"""
values = _indexed_metadata_values(metadata)
for key, value in values.items():
stmt = stmt.where(_INDEXED_METADATA_COLUMNS[key] == value)
return stmt, set(values)
class ThreadMetaRepository(ThreadMetaStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
@@ -40,18 +97,20 @@ class ThreadMetaRepository(ThreadMetaStore):
# creates an orphan row (used by migration scripts).
resolved_user_id = resolve_user_id(user_id, method_name="ThreadMetaRepository.create")
now = datetime.now(UTC)
row = ThreadMetaRow(
thread_id=thread_id,
assistant_id=assistant_id,
user_id=resolved_user_id,
display_name=display_name,
status="idle",
metadata_json=metadata or {},
row = ThreadMetaRow(
thread_id=thread_id,
assistant_id=assistant_id,
user_id=resolved_user_id,
**_indexed_thread_metadata(metadata),
display_name=display_name,
status="idle",
metadata_json=metadata or {},
created_at=now,
updated_at=now,
)
async with self._sf() as session:
session.add(row)
await sync_task_deeplink_index(session, row)
await session.commit()
# Deliberately no session.refresh(): every column above is assigned
# explicitly and the session factory uses expire_on_commit=False, so
@@ -108,10 +167,10 @@ class ThreadMetaRepository(ThreadMetaStore):
md = row.metadata_json or {}
# 全局共享、不区分用户(任何登录用户都可读写/续聊):
# ① 任务深链对话(metadata.taskId);
# ② 圆桌会商系统线程(metadata.thread_type=="roundtable")——其 Step2/Step3
# 沙箱产物(md/流程图/报告)需让历史草稿或其他账号打开同一会商时也能查看,
# 与任务深链对话一致(线程 id 仅存在于草稿中,不会被无关账号发现)。
if md.get("taskId") or md.get("thread_type") == "roundtable":
# ② 圆桌 / 任务分析等系统编排线程(thread_type 见 SHARED_ACCESS_THREAD_TYPES)
# —— loopback 派活签发的 JWT 可能与建 thread 时 HTTP 上下文用户不是同一人,
# 若按属主拦截会把刚建好的总控 thread 打成 404 not accessible。
if is_shared_access_thread(md):
return True
if row.user_id is None:
return True
@@ -144,16 +203,24 @@ class ThreadMetaRepository(ThreadMetaStore):
stmt = stmt.where(ThreadMetaRow.user_id == resolved_user_id)
if status:
stmt = stmt.where(ThreadMetaRow.status == status)
if query and query.strip():
if query and query.strip():
# Case-insensitive title search. display_name is a real column, so
# this filter runs in SQL even when the JSON filters below fall
# back to Python-side scanning.
stmt = stmt.where(func.lower(ThreadMetaRow.display_name).contains(query.strip().lower()))
if not metadata and not exclude_system:
stmt = stmt.limit(limit).offset(offset)
async with self._sf() as session:
result = await session.execute(stmt)
stmt = stmt.where(func.lower(ThreadMetaRow.display_name).contains(query.strip().lower()))
# ``metadata_json`` is deliberately portable, so arbitrary filters
# still receive Python-side equality checks below. These three task
# identifiers are maintained projections and can safely narrow the
# SQL result first. A deep-link query that used to scan every thread
# now reads only that task's rows.
stmt, indexed_metadata_keys = _apply_indexed_metadata_filters(stmt, metadata)
remaining_metadata_keys = set(metadata or {}) - indexed_metadata_keys
if not remaining_metadata_keys and not exclude_system:
stmt = stmt.limit(limit).offset(offset)
async with self._sf() as session:
result = await session.execute(stmt)
return [self._row_to_dict(r) for r in result.scalars()]
# Metadata filters live inside the JSON blob, so rows are scanned in
@@ -205,12 +272,17 @@ class ThreadMetaRepository(ThreadMetaStore):
conditions.append(ThreadMetaRow.user_id == resolved_user_id)
if status:
conditions.append(ThreadMetaRow.status == status)
if query and query.strip():
conditions.append(func.lower(ThreadMetaRow.display_name).contains(query.strip().lower()))
if not metadata and not exclude_system:
stmt = select(func.count()).select_from(ThreadMetaRow)
for condition in conditions:
if query and query.strip():
conditions.append(func.lower(ThreadMetaRow.display_name).contains(query.strip().lower()))
indexed_metadata = _indexed_metadata_values(metadata)
for key, value in indexed_metadata.items():
conditions.append(_INDEXED_METADATA_COLUMNS[key] == value)
remaining_metadata_keys = set(metadata or {}) - set(indexed_metadata)
if not remaining_metadata_keys and not exclude_system:
stmt = select(func.count()).select_from(ThreadMetaRow)
for condition in conditions:
stmt = stmt.where(condition)
async with self._sf() as session:
return int((await session.execute(stmt)).scalar_one())
@@ -318,11 +390,14 @@ class ThreadMetaRepository(ThreadMetaStore):
task_shared = bool((row.metadata_json or {}).get("taskId"))
if resolved_user_id is not None and not task_shared and row.user_id != resolved_user_id:
return
merged = dict(row.metadata_json or {})
merged.update(metadata)
row.metadata_json = merged
row.updated_at = datetime.now(UTC)
await session.commit()
merged = dict(row.metadata_json or {})
merged.update(metadata)
row.metadata_json = merged
for key, value in _indexed_thread_metadata(merged).items():
setattr(row, key, value)
row.updated_at = datetime.now(UTC)
await sync_task_deeplink_index(session, row)
await session.commit()
async def delete(
self,
@@ -339,5 +414,6 @@ class ThreadMetaRepository(ThreadMetaStore):
task_shared = bool((row.metadata_json or {}).get("taskId"))
if resolved_user_id is not None and not task_shared and row.user_id != resolved_user_id:
return
await delete_task_deeplink_index(session, thread_id)
await session.delete(row)
await session.commit()