fix: restore embedded wiki images and optimize vector sync

This commit is contained in:
吉鑫升 committed 2026-08-25 11:24:01 +08:00
1 parent 30943159bf
commit 0c5e2e8cb0
23 files changed
+609 -136

No files matched your search

@@ -6,8 +6,12 @@ from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWiki
from deerflow.persistence.llmwiki_index.sql import SqlLlmWikiIndexStore
def make_llmwiki_index_store(session_factory):
return MemoryLlmWikiIndexStore() if session_factory is None else SqlLlmWikiIndexStore(session_factory)
def make_llmwiki_index_store(session_factory, *, write_batch_size: int = 50):
return (
MemoryLlmWikiIndexStore()
if session_factory is None
else SqlLlmWikiIndexStore(session_factory, write_batch_size=write_batch_size)
)
__all__ = [
@@ -6,7 +6,7 @@ from datetime import UTC, datetime, timedelta
from typing import Any
from uuid import uuid4
from sqlalchemy import case, delete, func, or_, select, update
from sqlalchemy import case, delete, func, insert, or_, select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
@@ -27,8 +27,9 @@ def _dict(row) -> dict[str, Any]:
class SqlLlmWikiIndexStore(LlmWikiIndexStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
def __init__(self, session_factory: async_sessionmaker[AsyncSession], *, write_batch_size: int = 50) -> None:
self._sf = session_factory
self.write_batch_size = max(1, min(int(write_batch_size), 500))
async def _ensure_state(self, session: AsyncSession, mapping_id: str) -> LlmWikiWikiSyncStateRow:
row = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
@@ -82,24 +83,29 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
row.updated_at = now
await session.flush()
await session.execute(delete(LlmWikiWikiVectorRow).where(LlmWikiWikiVectorRow.page_id == row.id))
session.add_all(
[
LlmWikiWikiVectorRow(
id=str(uuid4()),
page_id=row.id,
knowledge_base_mapping_id=mapping_id,
section_index=int(item["section_index"]),
heading=item.get("heading"),
section_content=str(item["section_content"]),
content_hash=str(item["content_hash"]),
embedding_fingerprint=fingerprint,
embedding_dimensions=int(item["embedding_dimensions"]),
vector_blob=bytes(item["vector_blob"]),
created_at=now,
)
for item in vectors
]
)
vector_rows = [
{
"id": str(uuid4()),
"page_id": row.id,
"knowledge_base_mapping_id": mapping_id,
"section_index": int(item["section_index"]),
"heading": item.get("heading"),
"section_content": str(item["section_content"]),
"content_hash": str(item["content_hash"]),
"embedding_fingerprint": fingerprint,
"embedding_dimensions": int(item["embedding_dimensions"]),
"vector_blob": bytes(item["vector_blob"]),
"created_at": now,
}
for item in vectors
]
# Keep the page replacement atomic, but cap each SQL statement so
# large Wiki pages cannot flood the database with one huge insert.
for start in range(0, len(vector_rows), self.write_batch_size):
await session.execute(
insert(LlmWikiWikiVectorRow),
vector_rows[start : start + self.write_batch_size],
)
state = await self._ensure_state(session, mapping_id)
state.index_revision = int(state.index_revision or 0) + 1
state.embedding_fingerprint = fingerprint