fix: harden wiki delivery and leaderboard performance

This commit is contained in:
吉鑫升 committed 2026-08-25 16:58:13 +08:00
1 parent 4b86106a24
commit e053b36b6e
28 files changed
+616 -68

No files matched your search

@@ -44,11 +44,24 @@ class LlmWikiIndexStore(ABC):
@abstractmethod
async def renew_sync_lease(self, mapping_id: str, *, owner: str, lease_seconds: int) -> bool: ...
@abstractmethod
async def update_sync_progress(
self,
mapping_id: str,
*,
owner: str,
sync_id: str,
remote_page_count: int,
) -> bool: ...
@abstractmethod
async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None: ...
@abstractmethod
async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]: ...
@abstractmethod
async def list_page_index_status(self, mapping_id: str) -> list[dict[str, Any]]: ...
@abstractmethod
async def mark_rebuild_required(self, mapping_ids: list[str], fingerprint: str) -> int: ...
@@ -123,6 +123,21 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
state["lease_expires_at"] = _now() + timedelta(seconds=lease_seconds)
return True
async def update_sync_progress(
self,
mapping_id: str,
*,
owner: str,
sync_id: str,
remote_page_count: int,
) -> bool:
state = self._state(mapping_id)
if state.get("lease_owner") != owner or state.get("active_sync_id") != sync_id:
return False
state["remote_page_count"] = max(0, remote_page_count)
state["updated_at"] = _now()
return True
async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None:
state = self._state(mapping_id)
if state.get("lease_owner") != owner or state.get("active_sync_id") != sync_id:
@@ -150,6 +165,33 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
wanted = set(mapping_ids or [])
return [dict(value) for key, value in self.states.items() if not wanted or key in wanted]
async def list_page_index_status(self, mapping_id: str) -> list[dict[str, Any]]:
result = []
for page in sorted(
(
page
for page in self.pages.values()
if page["knowledge_base_mapping_id"] == mapping_id and not page.get("is_remote_deleted")
),
key=lambda page: (str(page.get("wiki_path") or ""), str(page.get("slug") or "")),
):
result.append(
{
"id": page["id"],
"slug": page["slug"],
"title": page["title"],
"page_type": page["page_type"],
"status": page["status"],
"index_status": page["index_status"],
"vector_count": len(self.vectors.get(page["id"], [])),
"indexed_at": page.get("indexed_at"),
"last_error": page.get("last_error"),
"last_seen_sync_id": page.get("last_seen_sync_id"),
"updated_at": page.get("updated_at"),
}
)
return result
async def mark_rebuild_required(self, mapping_ids: list[str], fingerprint: str) -> int:
for mapping_id in mapping_ids:
state = self._state(mapping_id)
@@ -235,6 +235,27 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
await session.commit()
return bool(result.rowcount)
async def update_sync_progress(
self,
mapping_id: str,
*,
owner: str,
sync_id: str,
remote_page_count: int,
) -> bool:
async with self._sf() as session:
result = await session.execute(
update(LlmWikiWikiSyncStateRow)
.where(
LlmWikiWikiSyncStateRow.knowledge_base_mapping_id == mapping_id,
LlmWikiWikiSyncStateRow.lease_owner == owner,
LlmWikiWikiSyncStateRow.active_sync_id == sync_id,
)
.values(remote_page_count=max(0, remote_page_count), updated_at=_now())
)
await session.commit()
return bool(result.rowcount)
async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None:
async with self._sf() as session:
state = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
@@ -286,6 +307,47 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
async with self._sf() as session:
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def list_page_index_status(self, mapping_id: str) -> list[dict[str, Any]]:
vector_counts = (
select(
LlmWikiWikiVectorRow.page_id.label("page_id"),
func.count(LlmWikiWikiVectorRow.id).label("vector_count"),
)
.where(LlmWikiWikiVectorRow.knowledge_base_mapping_id == mapping_id)
.group_by(LlmWikiWikiVectorRow.page_id)
.subquery()
)
stmt = (
select(LlmWikiWikiPageRow, func.coalesce(vector_counts.c.vector_count, 0))
.outerjoin(vector_counts, vector_counts.c.page_id == LlmWikiWikiPageRow.id)
.where(
LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id,
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
)
.order_by(LlmWikiWikiPageRow.wiki_path, LlmWikiWikiPageRow.slug)
)
async with self._sf() as session:
rows = (await session.execute(stmt)).all()
result: list[dict[str, Any]] = []
for row, vector_count in rows:
page = _dict(row)
result.append(
{
"id": page["id"],
"slug": page["slug"],
"title": page["title"],
"page_type": page["page_type"],
"status": page["status"],
"index_status": page["index_status"],
"vector_count": int(vector_count or 0),
"indexed_at": page.get("indexed_at"),
"last_error": page.get("last_error"),
"last_seen_sync_id": page.get("last_seen_sync_id"),
"updated_at": page.get("updated_at"),
}
)
return result
async def mark_rebuild_required(self, mapping_ids: list[str], fingerprint: str) -> int:
if not mapping_ids:
return 0