sync update from LAN repo (source dhk-1 @ 3b1ec5b4)

This commit is contained in:
gozeros committed 2026-11-05 10:56:34 +08:00
1 parent da6521b6cb
commit 264e2200d6
995 files changed
+148105 -13399

No files matched your search

@@ -5,72 +5,60 @@ from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import func, select, update
from sqlalchemy import func, or_, select, update
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.json_compat import JsonPredicate, supports_json_filter
from deerflow.persistence.task_deeplink_threads.sql import (
delete_task_deeplink_index,
sync_task_deeplink_index,
update_task_deeplink_card,
)
from deerflow.persistence.thread_meta.base import ThreadMetaStore, is_excluded_system_thread, is_shared_access_thread
from deerflow.persistence.thread_meta.base import EXCLUDED_SYSTEM_THREAD_TYPES, ThreadMetaStore, is_excluded_system_thread, is_shared_access_thread
from deerflow.persistence.thread_meta.model import ThreadMetaRow
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
_INDEXED_METADATA_COLUMNS = {
"taskId": ThreadMetaRow.task_id,
"agent_id": ThreadMetaRow.agent_id,
"actionId": ThreadMetaRow.action_id,
}
_INDEXED_METADATA_ATTRIBUTES = {
"taskId": "task_id",
"agent_id": "agent_id",
"actionId": "action_id",
}
def _indexed_metadata_values(metadata: dict[str, Any] | None) -> dict[str, str]:
"""Return metadata predicates that have a maintained SQL projection.
Task deep-link callers use string ids. Restrict the SQL shortcut to
strings so legacy callers that intentionally store another JSON type keep
the exact Python-side equality semantics of the generic metadata filter.
"""
return {
key: value
for key, value in (metadata or {}).items()
if key in _INDEXED_METADATA_COLUMNS and isinstance(value, str)
}
def _indexed_thread_metadata(metadata: dict[str, Any] | None) -> dict[str, str | None]:
"""Build ORM column values from authoritative thread metadata.
Always include every projection so a metadata update that changes a value
from a string to another JSON type cannot leave a stale indexed value.
"""
values = _indexed_metadata_values(metadata)
return {
attribute: values.get(metadata_key)
for metadata_key, attribute in _INDEXED_METADATA_ATTRIBUTES.items()
}
def _apply_indexed_metadata_filters(stmt: Any, metadata: dict[str, Any] | None) -> tuple[Any, set[str]]:
"""Apply known metadata predicates in SQL and return their keys.
The caller still verifies the full metadata object in Python whenever a
generic filter or system-thread exclusion is involved. This keeps the
public API's exact-match contract while reducing that verification set to
rows for the requested task/agent/action instead of the entire table.
"""
values = _indexed_metadata_values(metadata)
for key, value in values.items():
stmt = stmt.where(_INDEXED_METADATA_COLUMNS[key] == value)
return stmt, set(values)
class ThreadMetaRepository(ThreadMetaStore):
_INDEXED_METADATA_COLUMNS = {
"taskId": ThreadMetaRow.task_id,
"agent_id": ThreadMetaRow.agent_id,
"actionId": ThreadMetaRow.action_id,
}
_INDEXED_METADATA_ATTRIBUTES = {
"taskId": "task_id",
"agent_id": "agent_id",
"actionId": "action_id",
}
def _indexed_metadata_values(metadata: dict[str, Any] | None) -> dict[str, str]:
"""Return metadata predicates that have a maintained SQL projection.
Task deep-link callers use string ids. Restrict the SQL shortcut to
strings so legacy callers that intentionally store another JSON type keep
the exact Python-side equality semantics of the generic metadata filter.
"""
return {
key: value
for key, value in (metadata or {}).items()
if key in _INDEXED_METADATA_COLUMNS and isinstance(value, str)
}
def _indexed_thread_metadata(metadata: dict[str, Any] | None) -> dict[str, str | None]:
"""Build ORM column values from authoritative thread metadata.
Always include every projection so a metadata update that changes a value
from a string to another JSON type cannot leave a stale indexed value.
"""
values = _indexed_metadata_values(metadata)
return {
attribute: values.get(metadata_key)
for metadata_key, attribute in _INDEXED_METADATA_ATTRIBUTES.items()
}
class ThreadMetaRepository(ThreadMetaStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
@@ -97,14 +85,14 @@ class ThreadMetaRepository(ThreadMetaStore):
# creates an orphan row (used by migration scripts).
resolved_user_id = resolve_user_id(user_id, method_name="ThreadMetaRepository.create")
now = datetime.now(UTC)
row = ThreadMetaRow(
thread_id=thread_id,
assistant_id=assistant_id,
user_id=resolved_user_id,
**_indexed_thread_metadata(metadata),
display_name=display_name,
status="idle",
metadata_json=metadata or {},
row = ThreadMetaRow(
thread_id=thread_id,
assistant_id=assistant_id,
user_id=resolved_user_id,
**_indexed_thread_metadata(metadata),
display_name=display_name,
status="idle",
metadata_json=metadata or {},
created_at=now,
updated_at=now,
)
@@ -164,6 +152,10 @@ class ThreadMetaRepository(ThreadMetaStore):
row = await session.get(ThreadMetaRow, thread_id)
if row is None:
return not require_existing
if row.user_id:
from deerflow.runtime.thread_paths import remember_thread_owner
remember_thread_owner(thread_id, row.user_id)
md = row.metadata_json or {}
# 全局共享、不区分用户(任何登录用户都可读写/续聊):
# ① 任务深链对话(metadata.taskId);
@@ -176,6 +168,43 @@ class ThreadMetaRepository(ThreadMetaStore):
return True
return row.user_id == user_id
@staticmethod
def _query_conditions(*, dialect, metadata, status, user_id, exclude_system, query):
"""Share exact search/count predicates without changing owner resolution.
Complex metadata and unsupported dialects retain their exact Python
filter. No rejected key/value is silently ignored. Server numeric JSON
comparisons retain that path: DECIMAL and DOUBLE cannot reproduce all
Python float/int comparisons (including large numeric task ids).
"""
conditions = []
if user_id is not None:
conditions.append(ThreadMetaRow.user_id == user_id)
if status:
conditions.append(ThreadMetaRow.status == status)
if query and query.strip():
conditions.append(func.lower(ThreadMetaRow.display_name).contains(query.strip().lower()))
# Retain the master branch's indexed task/agent/action narrowing.
# JSON predicates below still enforce exact case/type semantics.
for key, value in _indexed_metadata_values(metadata).items():
conditions.append(_INDEXED_METADATA_COLUMNS[key] == value)
supported = dialect in {"sqlite", "postgresql", "mysql"}
residual = {}
for key, value in (metadata or {}).items():
numeric_server = dialect in {"mysql", "postgresql"} and isinstance(value, (bool, int, float))
if supported and supports_json_filter(key, value) and not numeric_server:
conditions.append(JsonPredicate(ThreadMetaRow.metadata_json, key, value=value))
else:
residual[key] = value
if exclude_system and supported:
column = ThreadMetaRow.metadata_json
conditions.append(~or_(
JsonPredicate(column, "system", "truthy"),
JsonPredicate(column, "task_analysis_seat", "nonblank"),
*(JsonPredicate(column, "thread_type", value=value) for value in sorted(EXCLUDED_SYSTEM_THREAD_TYPES)),
))
return conditions, residual, exclude_system and not supported
async def search(
self,
*,
@@ -187,71 +216,46 @@ class ThreadMetaRepository(ThreadMetaStore):
exclude_system: bool = False,
query: str | None = None,
) -> list[dict]:
"""Search threads with optional metadata and status filters.
Owner filter is enforced by default: caller must be in a user
context. Pass ``user_id=None`` to bypass (migration/CLI).
"""
resolved_user_id = resolve_user_id(user_id, method_name="ThreadMetaRepository.search")
# 任务深链对话(按 metadata.taskId 过滤)全局共享、不区分用户:跳过 owner 过滤。
# Preserve task-scoped sharing, still applying EVERY metadata filter
# (taskId + agent_id + actionId) before the pagination window.
if metadata and metadata.get("taskId"):
resolved_user_id = None
# thread_id tie-breaker keeps OFFSET pagination stable when many rows
# share the same updated_at (bulk imports, scheduler bursts).
stmt = select(ThreadMetaRow).order_by(ThreadMetaRow.updated_at.desc(), ThreadMetaRow.thread_id.desc())
if resolved_user_id is not None:
stmt = stmt.where(ThreadMetaRow.user_id == resolved_user_id)
if status:
stmt = stmt.where(ThreadMetaRow.status == status)
if query and query.strip():
# Case-insensitive title search. display_name is a real column, so
# this filter runs in SQL even when the JSON filters below fall
# back to Python-side scanning.
stmt = stmt.where(func.lower(ThreadMetaRow.display_name).contains(query.strip().lower()))
# ``metadata_json`` is deliberately portable, so arbitrary filters
# still receive Python-side equality checks below. These three task
# identifiers are maintained projections and can safely narrow the
# SQL result first. A deep-link query that used to scan every thread
# now reads only that task's rows.
stmt, indexed_metadata_keys = _apply_indexed_metadata_filters(stmt, metadata)
remaining_metadata_keys = set(metadata or {}) - indexed_metadata_keys
if not remaining_metadata_keys and not exclude_system:
stmt = stmt.limit(limit).offset(offset)
async with self._sf() as session:
result = await session.execute(stmt)
return [self._row_to_dict(r) for r in result.scalars()]
# Metadata filters live inside the JSON blob, so rows are scanned in
# batches and filtered in Python until the requested window is full.
# A fixed over-fetch window would under-fill pages when most rows are
# filtered out (e.g. heavy scheduled-task usage), making callers that
# paginate believe the list is exhausted. TODO(Phase 2): use JSON DB
# operators (Postgres @>, SQLite json_extract) for server-side filtering.
def _matches(row: dict) -> bool:
md = row.get("metadata") or {}
if metadata and not all(md.get(k) == v for k, v in metadata.items()):
return False
if exclude_system and is_excluded_system_thread(md):
return False
return True
needed = offset + limit
batch_size = max(500, needed)
matched: list[dict] = []
scan_offset = 0
async with self._sf() as session:
conditions, residual, filter_system = self._query_conditions(
dialect=session.get_bind().dialect.name,
metadata=metadata, status=status, user_id=resolved_user_id,
exclude_system=exclude_system, query=query,
)
stmt = select(ThreadMetaRow).where(*conditions).order_by(
ThreadMetaRow.updated_at.desc(), ThreadMetaRow.thread_id.desc(),
)
if not residual and not filter_system:
result = await session.execute(stmt.limit(limit).offset(offset))
return [self._row_to_dict(row) for row in result.scalars()]
# Compatibility path for nested values, unusual literal keys,
# numeric server filters, or the separate openGauss dialect.
needed = offset + limit
batch_size = max(500, needed)
matched = []
scan_offset = 0
while len(matched) < needed:
result = await session.execute(stmt.limit(batch_size).offset(scan_offset))
rows = result.scalars().all()
if not rows:
break
matched.extend(d for d in (self._row_to_dict(r) for r in rows) if _matches(d))
for row in rows:
record = self._row_to_dict(row)
md = record.get("metadata") or {}
if all(md.get(key) == value for key, value in residual.items()) and not (
filter_system and is_excluded_system_thread(md)
):
matched.append(record)
if len(rows) < batch_size:
break
scan_offset += len(rows)
return matched[offset:needed]
return matched[offset:needed]
async def count(
self,
@@ -262,56 +266,37 @@ class ThreadMetaRepository(ThreadMetaStore):
exclude_system: bool = False,
query: str | None = None,
) -> int:
"""Count threads matching the same filters as :meth:`search`."""
resolved_user_id = resolve_user_id(user_id, method_name="ThreadMetaRepository.count")
# 任务深链对话(按 metadata.taskId 过滤)全局共享、不区分用户:跳过 owner 过滤。
if metadata and metadata.get("taskId"):
resolved_user_id = None
conditions = []
if resolved_user_id is not None:
conditions.append(ThreadMetaRow.user_id == resolved_user_id)
if status:
conditions.append(ThreadMetaRow.status == status)
if query and query.strip():
conditions.append(func.lower(ThreadMetaRow.display_name).contains(query.strip().lower()))
indexed_metadata = _indexed_metadata_values(metadata)
for key, value in indexed_metadata.items():
conditions.append(_INDEXED_METADATA_COLUMNS[key] == value)
remaining_metadata_keys = set(metadata or {}) - set(indexed_metadata)
if not remaining_metadata_keys and not exclude_system:
stmt = select(func.count()).select_from(ThreadMetaRow)
for condition in conditions:
stmt = stmt.where(condition)
async with self._sf() as session:
async with self._sf() as session:
conditions, residual, filter_system = self._query_conditions(
dialect=session.get_bind().dialect.name,
metadata=metadata, status=status, user_id=resolved_user_id,
exclude_system=exclude_system, query=query,
)
if not residual and not filter_system:
stmt = select(func.count()).select_from(ThreadMetaRow).where(*conditions)
return int((await session.execute(stmt)).scalar_one())
# JSON filters can't run in portable SQL — scan just the metadata
# column in batches and count in Python (mirrors search()).
stmt = select(ThreadMetaRow.metadata_json).order_by(ThreadMetaRow.thread_id)
for condition in conditions:
stmt = stmt.where(condition)
total = 0
batch_size = 1000
scan_offset = 0
async with self._sf() as session:
stmt = select(ThreadMetaRow.metadata_json).where(*conditions).order_by(ThreadMetaRow.thread_id)
total = 0
scan_offset = 0
batch_size = 1000
while True:
result = await session.execute(stmt.limit(batch_size).offset(scan_offset))
rows = result.scalars().all()
rows = (await session.execute(stmt.limit(batch_size).offset(scan_offset))).scalars().all()
if not rows:
break
for metadata_json in rows:
md = metadata_json or {}
if metadata and not all(md.get(k) == v for k, v in metadata.items()):
continue
if exclude_system and is_excluded_system_thread(md):
continue
total += 1
for md in rows:
md = md or {}
if all(md.get(key) == value for key, value in residual.items()) and not (
filter_system and is_excluded_system_thread(md)
):
total += 1
if len(rows) < batch_size:
break
scan_offset += len(rows)
return total
return total
async def _check_ownership(self, session: AsyncSession, thread_id: str, resolved_user_id: str | None) -> bool:
"""Return True if the row exists and is owned (or filter bypassed)."""
@@ -337,7 +322,9 @@ class ThreadMetaRepository(ThreadMetaStore):
async with self._sf() as session:
if not await self._check_ownership(session, thread_id, resolved_user_id):
return
await session.execute(update(ThreadMetaRow).where(ThreadMetaRow.thread_id == thread_id).values(display_name=display_name, updated_at=datetime.now(UTC)))
now = datetime.now(UTC)
await session.execute(update(ThreadMetaRow).where(ThreadMetaRow.thread_id == thread_id).values(display_name=display_name, updated_at=now))
await update_task_deeplink_card(session, thread_id, display_name=display_name, updated_at=now)
await session.commit()
async def update_status(
@@ -351,7 +338,9 @@ class ThreadMetaRepository(ThreadMetaStore):
async with self._sf() as session:
if not await self._check_ownership(session, thread_id, resolved_user_id):
return
await session.execute(update(ThreadMetaRow).where(ThreadMetaRow.thread_id == thread_id).values(status=status, updated_at=datetime.now(UTC)))
now = datetime.now(UTC)
await session.execute(update(ThreadMetaRow).where(ThreadMetaRow.thread_id == thread_id).values(status=status, updated_at=now))
await update_task_deeplink_card(session, thread_id, status=status, updated_at=now)
await session.commit()
async def update_assistant_id(
@@ -365,7 +354,9 @@ class ThreadMetaRepository(ThreadMetaStore):
async with self._sf() as session:
if not await self._check_ownership(session, thread_id, resolved_user_id):
return
await session.execute(update(ThreadMetaRow).where(ThreadMetaRow.thread_id == thread_id).values(assistant_id=assistant_id, updated_at=datetime.now(UTC)))
now = datetime.now(UTC)
await session.execute(update(ThreadMetaRow).where(ThreadMetaRow.thread_id == thread_id).values(assistant_id=assistant_id, updated_at=now))
await update_task_deeplink_card(session, thread_id, updated_at=now)
await session.commit()
async def update_metadata(
@@ -390,14 +381,14 @@ class ThreadMetaRepository(ThreadMetaStore):
task_shared = bool((row.metadata_json or {}).get("taskId"))
if resolved_user_id is not None and not task_shared and row.user_id != resolved_user_id:
return
merged = dict(row.metadata_json or {})
merged.update(metadata)
row.metadata_json = merged
for key, value in _indexed_thread_metadata(merged).items():
setattr(row, key, value)
row.updated_at = datetime.now(UTC)
merged = dict(row.metadata_json or {})
merged.update(metadata)
row.metadata_json = merged
for key, value in _indexed_thread_metadata(merged).items():
setattr(row, key, value)
row.updated_at = datetime.now(UTC)
await sync_task_deeplink_index(session, row)
await session.commit()
await session.commit()
async def delete(
self,