Merge branch 'ji'

This commit is contained in:
吉鑫升 committed 2026-06-11 19:59:58 +08:00
commit 82600a3404
25 files changed
+929 -279

No files matched your search

@@ -28,6 +28,7 @@ from deerflow.persistence.feedback.model import FeedbackRow
from deerflow.knowledge.models import (
KnowledgeEmbeddingRow,
KnowledgeEntityRow,
KnowledgeGraphBlacklistRow,
KnowledgeNoteRow,
KnowledgeNoteTagRow,
KnowledgeNoteVersionRow,
@@ -75,6 +76,7 @@ __all__ = [
"BusinessMappingRow",
"KnowledgeEmbeddingRow",
"KnowledgeEntityRow",
"KnowledgeGraphBlacklistRow",
"KnowledgeNoteRow",
"KnowledgeNoteTagRow",
"KnowledgeNoteVersionRow",
@@ -45,7 +45,18 @@ class ThreadMetaStore(abc.ABC):
limit: int = 100,
offset: int = 0,
user_id: str | None | _AutoSentinel = AUTO,
exclude_system: bool = False,
query: str | None = None,
) -> list[dict]:
"""Search threads ordered by ``updated_at`` desc.
``exclude_system=True`` drops threads whose ``metadata.system`` is
truthy (scheduler runs, roundtable workers, …) so user-facing
conversation lists are not crowded out by machine-created threads.
``query`` is a case-insensitive substring match on the thread title
(``display_name``). ``limit``/``offset`` apply to the *filtered*
result set.
"""
pass
@abc.abstractmethod
@@ -74,6 +74,8 @@ class MemoryThreadMetaStore(ThreadMetaStore):
limit: int = 100,
offset: int = 0,
user_id: str | None | _AutoSentinel = AUTO,
exclude_system: bool = False,
query: str | None = None,
) -> list[dict]:
resolved_user_id = resolve_user_id(user_id, method_name="MemoryThreadMetaStore.search")
filter_dict: dict[str, Any] = {}
@@ -84,13 +86,45 @@ class MemoryThreadMetaStore(ThreadMetaStore):
if resolved_user_id is not None:
filter_dict["user_id"] = resolved_user_id
items = await self._store.asearch(
THREADS_NS,
filter=filter_dict or None,
limit=limit,
offset=offset,
)
return [self._item_to_dict(item) for item in items]
normalized_query = (query or "").strip().lower()
if not exclude_system and not normalized_query:
items = await self._store.asearch(
THREADS_NS,
filter=filter_dict or None,
limit=limit,
offset=offset,
)
return [self._item_to_dict(item) for item in items]
# The Store filter only supports equality, so the system-thread
# exclusion and title search are applied in Python: scan in batches
# until the requested window is full so ``limit``/``offset`` refer to
# filtered rows.
def _matches(record: dict) -> bool:
if exclude_system and (record.get("metadata") or {}).get("system"):
return False
if normalized_query and normalized_query not in (record.get("display_name") or "").lower():
return False
return True
needed = offset + limit
batch_size = max(500, needed)
matched: list[dict] = []
scan_offset = 0
while len(matched) < needed:
items = await self._store.asearch(
THREADS_NS,
filter=filter_dict or None,
limit=batch_size,
offset=scan_offset,
)
if not items:
break
matched.extend(record for record in (self._item_to_dict(item) for item in items) if _matches(record))
if len(items) < batch_size:
break
scan_offset += len(items)
return matched[offset:needed]
async def check_access(self, thread_id: str, user_id: str, *, require_existing: bool = False) -> bool:
item = await self._store.aget(THREADS_NS, thread_id)
@@ -5,7 +5,7 @@ from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import select, update
from sqlalchemy import func, select, update
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.thread_meta.base import ThreadMetaStore
@@ -109,6 +109,8 @@ class ThreadMetaRepository(ThreadMetaStore):
limit: int = 100,
offset: int = 0,
user_id: str | None | _AutoSentinel = AUTO,
exclude_system: bool = False,
query: str | None = None,
) -> list[dict]:
"""Search threads with optional metadata and status filters.
@@ -116,28 +118,55 @@ class ThreadMetaRepository(ThreadMetaStore):
context. Pass ``user_id=None`` to bypass (migration/CLI).
"""
resolved_user_id = resolve_user_id(user_id, method_name="ThreadMetaRepository.search")
stmt = select(ThreadMetaRow).order_by(ThreadMetaRow.updated_at.desc())
# thread_id tie-breaker keeps OFFSET pagination stable when many rows
# share the same updated_at (bulk imports, scheduler bursts).
stmt = select(ThreadMetaRow).order_by(ThreadMetaRow.updated_at.desc(), ThreadMetaRow.thread_id.desc())
if resolved_user_id is not None:
stmt = stmt.where(ThreadMetaRow.user_id == resolved_user_id)
if status:
stmt = stmt.where(ThreadMetaRow.status == status)
if query and query.strip():
# Case-insensitive title search. display_name is a real column, so
# this filter runs in SQL even when the JSON filters below fall
# back to Python-side scanning.
stmt = stmt.where(func.lower(ThreadMetaRow.display_name).contains(query.strip().lower()))
if metadata:
# When metadata filter is active, fetch a larger window and filter
# in Python. TODO(Phase 2): use JSON DB operators (Postgres @>,
# SQLite json_extract) for server-side filtering.
stmt = stmt.limit(limit * 5 + offset)
async with self._sf() as session:
result = await session.execute(stmt)
rows = [self._row_to_dict(r) for r in result.scalars()]
rows = [r for r in rows if all(r.get("metadata", {}).get(k) == v for k, v in metadata.items())]
return rows[offset : offset + limit]
else:
if not metadata and not exclude_system:
stmt = stmt.limit(limit).offset(offset)
async with self._sf() as session:
result = await session.execute(stmt)
return [self._row_to_dict(r) for r in result.scalars()]
# Metadata filters live inside the JSON blob, so rows are scanned in
# batches and filtered in Python until the requested window is full.
# A fixed over-fetch window would under-fill pages when most rows are
# filtered out (e.g. heavy scheduled-task usage), making callers that
# paginate believe the list is exhausted. TODO(Phase 2): use JSON DB
# operators (Postgres @>, SQLite json_extract) for server-side filtering.
def _matches(row: dict) -> bool:
md = row.get("metadata") or {}
if metadata and not all(md.get(k) == v for k, v in metadata.items()):
return False
if exclude_system and md.get("system"):
return False
return True
needed = offset + limit
batch_size = max(500, needed)
matched: list[dict] = []
scan_offset = 0
async with self._sf() as session:
while len(matched) < needed:
result = await session.execute(stmt.limit(batch_size).offset(scan_offset))
rows = result.scalars().all()
if not rows:
break
matched.extend(d for d in (self._row_to_dict(r) for r in rows) if _matches(d))
if len(rows) < batch_size:
break
scan_offset += len(rows)
return matched[offset:needed]
async def _check_ownership(self, session: AsyncSession, thread_id: str, resolved_user_id: str | None) -> bool:
"""Return True if the row exists and is owned (or filter bypassed)."""
if resolved_user_id is None: