fix: restore embedded wiki images and optimize vector sync

This commit is contained in:
吉鑫升 committed 2026-08-25 11:24:01 +08:00
1 parent 30943159bf
commit 0c5e2e8cb0
23 files changed
+609 -136

No files matched your search

@@ -40,20 +40,31 @@ async def test_sql_store_atomically_replaces_vectors_and_marks_missing_deleted(t
)
await session.commit()
store = SqlLlmWikiIndexStore(session_factory)
store = SqlLlmWikiIndexStore(session_factory, write_batch_size=1)
assert await store.try_acquire_sync_lease("mapping-1", owner="worker", sync_id="sync-1", lease_seconds=60, fingerprint="fp")
assert not await store.try_acquire_sync_lease("mapping-1", owner="worker", sync_id="sync-2", lease_seconds=60, fingerprint="fp")
page = normalize_wiki_page("mapping-1", {"id": "p1", "slug": "people/alice", "title": "Alice", "content": "Alice founded Acme.", "status": "published"})
blob, dimensions = encode_vector([3.0, 4.0], dimensions=2)
saved = await store.replace_page_vectors(
page=page,
vectors=[{"section_index": 0, "heading": "History", "section_content": "Alice founded Acme.", "content_hash": "section", "embedding_dimensions": dimensions, "vector_blob": blob}],
vectors=[
{
"section_index": index,
"heading": "History",
"section_content": f"Alice founded Acme. Part {index}",
"content_hash": f"section-{index}",
"embedding_dimensions": dimensions,
"vector_blob": blob,
}
for index in range(3)
],
fingerprint="fp",
sync_id="sync-1",
)
snapshot = await store.load_vector_snapshot("mapping-1", "fp")
assert saved["slug"] == "people/alice"
assert len(snapshot["rows"]) == 1
assert store.write_batch_size == 1
assert len(snapshot["rows"]) == 3
assert decode_vector(snapshot["rows"][0]["vector_blob"], 2).tolist() == pytest.approx([0.6, 0.8])
assert await store.mark_missing_pages_deleted("mapping-1", "sync-2") == 1
@@ -690,6 +690,24 @@ async def test_weknora_client_proxies_original_document_preview_without_credenti
assert media_type == "text/plain"
@pytest.mark.asyncio
async def test_weknora_client_fetches_protected_wiki_image() -> None:
async def handler(request: httpx.Request) -> httpx.Response:
assert request.url.path == "/api/v1/knowledge-bases/kb-one/files"
assert request.url.params["file_path"] == "resource://wiki/figure.png"
return httpx.Response(200, content=b"png-bytes", headers={"Content-Type": "image/png"})
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http_client:
client = WeKnoraClient(base_url="http://weknora.local:8080", http_client=http_client)
content, media_type = await client.get_knowledge_base_file(
"kb-one",
"resource://wiki/figure.png",
)
assert content == b"png-bytes"
assert media_type == "image/png"
@pytest.mark.asyncio
async def test_memory_store_enforces_owner_and_public_visibility() -> None:
store = MemoryLlmWikiStore()
@@ -1358,6 +1376,10 @@ def test_weknora_iframe_html_rewrites_public_prefix_paths() -> None:
assert '"allowedWeKnoraIds":["remote-kb","other-visible-kb"]' in injected
assert ".breadcrumb-link.dropdown" not in injected
assert "reloadForKnowledgeBaseSwitch" in injected
assert "function protectedImageApiUrl" in injected
assert 'img[data-protected-src]' in injected
assert 'encodeURIComponent(cfg.weknoraId)' in injected
assert 'credentials: "same-origin"' in injected
def test_weknora_iframe_filters_kb_list_to_signed_visible_mappings() -> None:
@@ -1,10 +1,13 @@
from __future__ import annotations
import asyncio
import json
from types import SimpleNamespace
import httpx
import pytest
from app.gateway import llmwiki_index_scheduler
from deerflow.config.llmwiki_config import LocalWikiEmbeddingConfig, LocalWikiIndexConfig
from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
@@ -85,6 +88,24 @@ async def test_configured_dimensions_are_validated_but_not_sent_to_bge() -> None
assert requests == [{"model": "bge-embedding-m3", "input": ["Wiki page"]}]
@pytest.mark.asyncio
async def test_embedding_error_includes_upstream_reason(caplog) -> None:
def handler(request: httpx.Request) -> httpx.Response:
return httpx.Response(400, json={"detail": "unsupported request parameter: dimensions"})
config = LocalWikiEmbeddingConfig(
model="bge-embedding-m3",
base_url="http://embedding.test/v1",
max_retries=0,
)
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http_client:
client = StrictWikiEmbeddingClient(config, http_client=http_client)
with pytest.raises(RuntimeError, match="unsupported request parameter: dimensions"):
await client.embed_query("Wiki page")
assert "unsupported request parameter: dimensions" in caplog.text
@pytest.mark.asyncio
async def test_omitted_dimensions_are_discovered_from_first_response() -> None:
def handler(request: httpx.Request) -> httpx.Response:
@@ -167,3 +188,39 @@ async def test_fingerprint_change_automatically_rebuilds_entire_mapping() -> Non
assert store.states["kb"]["state"] == "idle"
assert store.states["kb"]["embedding_fingerprint"] == "fp-new"
assert {vector["embedding_fingerprint"] for vectors in store.vectors.values() for vector in vectors} == {"fp-new"}
@pytest.mark.asyncio
async def test_auto_sync_scheduler_waits_before_first_scan(monkeypatch) -> None:
calls: list[str] = []
async def fake_sleep(seconds: float) -> None:
calls.append(f"sleep:{seconds}")
raise asyncio.CancelledError
async def fake_sync_once(app) -> None:
calls.append("sync")
monkeypatch.setattr(llmwiki_index_scheduler.asyncio, "sleep", fake_sleep)
monkeypatch.setattr(llmwiki_index_scheduler, "_sync_once", fake_sync_once)
app = SimpleNamespace(
state=SimpleNamespace(
config=SimpleNamespace(
llmwiki=SimpleNamespace(
local_wiki_index=SimpleNamespace(sync_interval_seconds=120),
)
)
)
)
with pytest.raises(asyncio.CancelledError):
await llmwiki_index_scheduler.local_wiki_index_scheduler_loop(app)
assert calls == ["sleep:120"]
def test_local_wiki_index_accepts_legacy_sync_flag() -> None:
config = LocalWikiIndexConfig.model_validate({"sync_enabled": True})
assert config.auto_sync is True
assert config.sync_enabled is True
@@ -177,6 +177,37 @@ async def test_public_wiki_page_list_is_normalized(monkeypatch) -> None:
assert payload["pages"][0]["aliases"] == []
@pytest.mark.asyncio
async def test_public_wiki_file_is_authorized_and_streamed(monkeypatch) -> None:
store = PublicStore(_published_row())
class Client:
async def get_knowledge_base_file(self, knowledge_base_id: str, file_path: str):
assert knowledge_base_id == "remote-one"
assert file_path == "storage://tenant/resource://wiki/figure.png"
return b"png-bytes", "image/png"
monkeypatch.setattr(llmwiki, "_client_or_503", lambda: Client())
response = await llmwiki.get_public_wiki_file(
_request(store),
"kb-one",
"storage://tenant/resource://wiki/figure.png",
)
assert store.calls == [("kb-one", "__public_wiki__", False, False)]
assert response.body == b"png-bytes"
assert response.media_type == "image/png"
assert response.headers["cache-control"] == "public, max-age=300"
def test_wiki_file_proxy_rejects_non_weknora_urls() -> None:
with pytest.raises(HTTPException) as exc_info:
llmwiki._validate_protected_wiki_file_path("https://example.test/private.png")
assert exc_info.value.status_code == 422
@pytest.mark.asyncio
async def test_public_wiki_index_preserves_weknora_group_and_item_order(monkeypatch) -> None:
store = PublicStore(_published_row())