feat: improve assistant knowledge import and database compatibility
This commit is contained in:
1 parent
fbd8dec48f
commit
6b724bd00d
51 files changed
+4393
-1021
No files matched your search
+18
@@ -3,6 +3,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any
|
||||
|
||||
|
||||
@@ -35,6 +36,23 @@ class LlmWikiIndexStore(ABC):
|
||||
@abstractmethod
|
||||
async def load_vector_snapshot(self, mapping_id: str, fingerprint: str) -> dict[str, Any]: ...
|
||||
|
||||
async def iter_vector_snapshot_rows(
|
||||
self,
|
||||
mapping_id: str,
|
||||
fingerprint: str,
|
||||
*,
|
||||
batch_size: int = 1000,
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
"""Iterate vector rows for streaming export.
|
||||
|
||||
Implementations with a database backend should override this method so
|
||||
very large vector packages do not need to be materialized in memory.
|
||||
"""
|
||||
snapshot = await self.load_vector_snapshot(mapping_id, fingerprint)
|
||||
for row in snapshot.get("rows") or []:
|
||||
if isinstance(row, dict):
|
||||
yield row
|
||||
|
||||
@abstractmethod
|
||||
async def get_index_revision(self, mapping_id: str) -> tuple[int, str | None, str]: ...
|
||||
|
||||
|
||||
+36
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import AsyncIterator
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
@@ -181,6 +182,41 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
rows = [{**_dict(vector), "page": _dict(page)} for vector, page in pairs]
|
||||
return {"revision": int(state.index_revision if state else 0), "fingerprint": fingerprint, "rows": rows}
|
||||
|
||||
async def iter_vector_snapshot_rows(
|
||||
self,
|
||||
mapping_id: str,
|
||||
fingerprint: str,
|
||||
*,
|
||||
batch_size: int = 1000,
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
batch = max(1, min(int(batch_size), 5000))
|
||||
offset = 0
|
||||
while True:
|
||||
stmt = (
|
||||
select(LlmWikiWikiVectorRow, LlmWikiWikiPageRow)
|
||||
.join(LlmWikiWikiPageRow, LlmWikiWikiPageRow.id == LlmWikiWikiVectorRow.page_id)
|
||||
.where(
|
||||
LlmWikiWikiVectorRow.knowledge_base_mapping_id == mapping_id,
|
||||
LlmWikiWikiVectorRow.embedding_fingerprint == fingerprint,
|
||||
LlmWikiWikiPageRow.index_status == "ready",
|
||||
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
|
||||
LlmWikiWikiPageRow.status.in_(["published", "draft"]),
|
||||
)
|
||||
.order_by(LlmWikiWikiVectorRow.page_id, LlmWikiWikiVectorRow.section_index)
|
||||
.limit(batch)
|
||||
.offset(offset)
|
||||
)
|
||||
async with self._sf() as session:
|
||||
pairs = list((await session.execute(stmt)).all())
|
||||
rows = [{**_dict(vector), "page": _dict(page)} for vector, page in pairs]
|
||||
if not rows:
|
||||
break
|
||||
for row in rows:
|
||||
yield row
|
||||
if len(rows) < batch:
|
||||
break
|
||||
offset += len(rows)
|
||||
|
||||
async def get_index_revision(self, mapping_id: str) -> tuple[int, str | None, str]:
|
||||
async with self._sf() as session:
|
||||
state = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
|
||||
|
||||
Reference in new issue
Block a user