fix: restore embedded wiki images and optimize vector sync
This commit is contained in:
1 parent
30943159bf
commit
0c5e2e8cb0
23 files changed
+609
-136
No files matched your search
+26
-20
@@ -6,7 +6,7 @@ from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import case, delete, func, or_, select, update
|
||||
from sqlalchemy import case, delete, func, insert, or_, select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
@@ -27,8 +27,9 @@ def _dict(row) -> dict[str, Any]:
|
||||
|
||||
|
||||
class SqlLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession], *, write_batch_size: int = 50) -> None:
|
||||
self._sf = session_factory
|
||||
self.write_batch_size = max(1, min(int(write_batch_size), 500))
|
||||
|
||||
async def _ensure_state(self, session: AsyncSession, mapping_id: str) -> LlmWikiWikiSyncStateRow:
|
||||
row = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
|
||||
@@ -82,24 +83,29 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
row.updated_at = now
|
||||
await session.flush()
|
||||
await session.execute(delete(LlmWikiWikiVectorRow).where(LlmWikiWikiVectorRow.page_id == row.id))
|
||||
session.add_all(
|
||||
[
|
||||
LlmWikiWikiVectorRow(
|
||||
id=str(uuid4()),
|
||||
page_id=row.id,
|
||||
knowledge_base_mapping_id=mapping_id,
|
||||
section_index=int(item["section_index"]),
|
||||
heading=item.get("heading"),
|
||||
section_content=str(item["section_content"]),
|
||||
content_hash=str(item["content_hash"]),
|
||||
embedding_fingerprint=fingerprint,
|
||||
embedding_dimensions=int(item["embedding_dimensions"]),
|
||||
vector_blob=bytes(item["vector_blob"]),
|
||||
created_at=now,
|
||||
)
|
||||
for item in vectors
|
||||
]
|
||||
)
|
||||
vector_rows = [
|
||||
{
|
||||
"id": str(uuid4()),
|
||||
"page_id": row.id,
|
||||
"knowledge_base_mapping_id": mapping_id,
|
||||
"section_index": int(item["section_index"]),
|
||||
"heading": item.get("heading"),
|
||||
"section_content": str(item["section_content"]),
|
||||
"content_hash": str(item["content_hash"]),
|
||||
"embedding_fingerprint": fingerprint,
|
||||
"embedding_dimensions": int(item["embedding_dimensions"]),
|
||||
"vector_blob": bytes(item["vector_blob"]),
|
||||
"created_at": now,
|
||||
}
|
||||
for item in vectors
|
||||
]
|
||||
# Keep the page replacement atomic, but cap each SQL statement so
|
||||
# large Wiki pages cannot flood the database with one huge insert.
|
||||
for start in range(0, len(vector_rows), self.write_batch_size):
|
||||
await session.execute(
|
||||
insert(LlmWikiWikiVectorRow),
|
||||
vector_rows[start : start + self.write_batch_size],
|
||||
)
|
||||
state = await self._ensure_state(session, mapping_id)
|
||||
state.index_revision = int(state.index_revision or 0) + 1
|
||||
state.embedding_fingerprint = fingerprint
|
||||
|
||||
Reference in new issue
Block a user