sync: 鍚屾灞€鍩熺綉鏈€鏂颁唬鐮?(source master @ b69568e4)
This commit is contained in:
1 parent
4f7f07504a
commit
8bc8b0e082
1651 files changed
+278599
-11416
No files matched your search
@@ -9,8 +9,13 @@ import pytest
|
||||
|
||||
from app.gateway import llmwiki_index_scheduler
|
||||
from deerflow.config.llmwiki_config import LocalWikiEmbeddingConfig, LocalWikiIndexConfig
|
||||
from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient
|
||||
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
|
||||
from deerflow.integrations.weknora.local_index import sync as wiki_sync
|
||||
from deerflow.integrations.weknora.local_index.embedding import (
|
||||
StrictWikiEmbeddingClient,
|
||||
WikiEmbeddingError,
|
||||
)
|
||||
from deerflow.integrations.weknora.local_index.normalize import normalize_wiki_page_links
|
||||
from deerflow.integrations.weknora.local_index.sync import WikiSyncLeaseLost, WikiSyncService
|
||||
from deerflow.integrations.weknora.local_index.vector_cache import WikiVectorCache
|
||||
from deerflow.persistence.llmwiki_index.memory import MemoryLlmWikiIndexStore
|
||||
|
||||
@@ -32,6 +37,50 @@ class FailingEmbedding(FakeEmbedding):
|
||||
raise RuntimeError("embedding unavailable")
|
||||
|
||||
|
||||
class UnavailableProviderEmbedding(FakeEmbedding):
|
||||
async def embed_texts(self, texts: list[str]) -> list[list[float]]:
|
||||
self.calls += 1
|
||||
raise WikiEmbeddingError(
|
||||
"connection refused",
|
||||
category="request",
|
||||
retryable=True,
|
||||
)
|
||||
|
||||
|
||||
class RecoverableProviderEmbedding(FakeEmbedding):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.available = False
|
||||
|
||||
async def embed_texts(self, texts: list[str]) -> list[list[float]]:
|
||||
self.calls += 1
|
||||
if not self.available:
|
||||
raise WikiEmbeddingError(
|
||||
"connection refused",
|
||||
category="request",
|
||||
retryable=True,
|
||||
)
|
||||
return [[1.0, 0.0] for _ in texts]
|
||||
|
||||
|
||||
class LeaseStealingEmbedding(FakeEmbedding):
|
||||
def __init__(self, store: MemoryLlmWikiIndexStore) -> None:
|
||||
super().__init__()
|
||||
self.store = store
|
||||
|
||||
async def embed_texts(self, texts: list[str]) -> list[list[float]]:
|
||||
state = self.store.states["kb"]
|
||||
state["lease_owner"] = "replacement-worker"
|
||||
state["active_sync_id"] = "replacement-sync"
|
||||
return await super().embed_texts(texts)
|
||||
|
||||
|
||||
class LeaseStealingFailingEmbedding(LeaseStealingEmbedding):
|
||||
async def embed_texts(self, texts: list[str]) -> list[list[float]]:
|
||||
await super().embed_texts(texts)
|
||||
raise RuntimeError("embedding failed after lease takeover")
|
||||
|
||||
|
||||
class NewFingerprintEmbedding(FakeEmbedding):
|
||||
fingerprint = "fp-new"
|
||||
|
||||
@@ -64,6 +113,91 @@ class FakeWeKnora:
|
||||
return dict(self.pages[slug])
|
||||
|
||||
|
||||
class ManyPageWeKnora(FakeWeKnora):
|
||||
def __init__(self, page_count: int = 50) -> None:
|
||||
super().__init__()
|
||||
self.pages = {
|
||||
f"page-{index}": {
|
||||
"id": f"p{index}",
|
||||
"slug": f"page-{index}",
|
||||
"title": f"Page {index}",
|
||||
"summary": "A long enough summary for indexing and retrieval.",
|
||||
"content": "# Intro\nA long enough Wiki paragraph for indexing and semantic retrieval.",
|
||||
"status": "published",
|
||||
"version": 1,
|
||||
}
|
||||
for index in range(page_count)
|
||||
}
|
||||
self.detail_calls = 0
|
||||
self.knowledge_base_calls = 0
|
||||
|
||||
async def get_knowledge_base(self, knowledge_base_id: str):
|
||||
self.knowledge_base_calls += 1
|
||||
return await super().get_knowledge_base(knowledge_base_id)
|
||||
|
||||
async def get_wiki_page(self, knowledge_base_id: str, slug: str):
|
||||
self.detail_calls += 1
|
||||
return await super().get_wiki_page(knowledge_base_id, slug)
|
||||
|
||||
|
||||
class LinkedFakeWeKnora(FakeWeKnora):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.pages["one"].update(
|
||||
{
|
||||
"out_links": [{"slug": "two"}],
|
||||
"content": "# Intro\nOne links to [[Two Page|the second page]].",
|
||||
}
|
||||
)
|
||||
self.pages["two"] = {
|
||||
"id": "p2",
|
||||
"slug": "two",
|
||||
"title": "Two Page",
|
||||
"summary": "A second page with enough text for the local index.",
|
||||
"content": "# Details\nThe second page contains connected evidence.",
|
||||
"inLinks": ["one"],
|
||||
"status": "published",
|
||||
"version": 1,
|
||||
}
|
||||
|
||||
async def get_wiki_graph(self, knowledge_base_id: str, *, limit: int):
|
||||
return {
|
||||
"nodes": [
|
||||
{"id": "node-one", "slug": "one", "title": "One"},
|
||||
{"id": "node-two", "slug": "two", "title": "Two Page"},
|
||||
],
|
||||
"edges": [{"source": "node-one", "target": "node-two"}],
|
||||
}
|
||||
|
||||
|
||||
class LeaseStealingGraphWeKnora(FakeWeKnora):
|
||||
def __init__(self, store: MemoryLlmWikiIndexStore) -> None:
|
||||
super().__init__()
|
||||
self.store = store
|
||||
|
||||
async def get_wiki_graph(self, knowledge_base_id: str, *, limit: int):
|
||||
del knowledge_base_id, limit
|
||||
state = self.store.states["kb"]
|
||||
state["lease_owner"] = "replacement-worker"
|
||||
state["active_sync_id"] = "replacement-sync"
|
||||
return {"nodes": [], "edges": []}
|
||||
|
||||
|
||||
def test_normalize_wiki_page_links_handles_provider_shapes_and_markdown() -> None:
|
||||
links = normalize_wiki_page_links(
|
||||
{
|
||||
"inLinks": [{"page_slug": "source/a#overview"}],
|
||||
"out_links": ["target/b", {"path": "target/c"}],
|
||||
"content": "See [[Target D#details|details]] and [[target/b]].",
|
||||
}
|
||||
)
|
||||
|
||||
assert links == {
|
||||
"in_links": ["source/a"],
|
||||
"out_links": ["Target D", "target/b", "target/c"],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_configured_dimensions_are_validated_but_not_sent_to_bge() -> None:
|
||||
requests: list[dict] = []
|
||||
@@ -131,6 +265,107 @@ async def test_omitted_dimensions_are_discovered_from_first_response() -> None:
|
||||
assert client.dimensions == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_provider_failure_aborts_remaining_pages_and_opens_backoff(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
now = 0.0
|
||||
monkeypatch.setattr(wiki_sync, "monotonic", lambda: now)
|
||||
config = LocalWikiIndexConfig(chunk_min_chars=10, min_similarity=0)
|
||||
store = MemoryLlmWikiIndexStore()
|
||||
embedding = UnavailableProviderEmbedding()
|
||||
client = ManyPageWeKnora()
|
||||
service = WikiSyncService(
|
||||
config,
|
||||
store,
|
||||
embedding,
|
||||
WikiVectorCache(store),
|
||||
client,
|
||||
)
|
||||
mapping = {"id": "kb", "weknora_id": "remote", "name": "Wiki"}
|
||||
|
||||
with pytest.raises(WikiEmbeddingError, match="connection refused"):
|
||||
await service.sync_mapping(mapping)
|
||||
|
||||
# The provider client's own retry policy is paid for one page only. The
|
||||
# remaining 49 pages are not fetched or embedded after a shared provider
|
||||
# failure.
|
||||
assert embedding.calls == 1
|
||||
assert client.detail_calls == 1
|
||||
assert store.states["kb"]["state"] == "failed"
|
||||
assert store.states["kb"]["last_sync_result"]["failure_scope"] == "embedding_provider"
|
||||
assert store.states["kb"]["last_sync_result"]["retry_after_seconds"] == 60
|
||||
|
||||
backoff = await service.sync_mapping(mapping)
|
||||
|
||||
assert backoff == {
|
||||
"mapping_id": "kb",
|
||||
"status": "provider_backoff",
|
||||
"failure_scope": "embedding_provider",
|
||||
"failure_count": 1,
|
||||
"retry_after_seconds": 60,
|
||||
"error": "WikiEmbeddingError: connection refused",
|
||||
}
|
||||
assert embedding.calls == 1
|
||||
assert client.knowledge_base_calls == 1
|
||||
|
||||
manual = await service.sync_mapping(mapping, sync_id="manual-sync")
|
||||
|
||||
assert manual["status"] == "failed"
|
||||
assert manual["sync_id"] == "manual-sync"
|
||||
assert manual["failure_scope"] == "embedding_provider"
|
||||
assert manual["failure_count"] == 1
|
||||
assert manual["retry_after_seconds"] == 60
|
||||
assert store.states["kb"]["state"] == "failed"
|
||||
assert store.states["kb"]["remote_page_count"] == 50
|
||||
assert store.states["kb"]["last_sync_result"] == manual
|
||||
assert embedding.calls == 1
|
||||
assert client.knowledge_base_calls == 1
|
||||
|
||||
# Once due, a single tiny provider probe fails before any second Wiki
|
||||
# directory scan and doubles the next breaker window.
|
||||
now = 60.0
|
||||
with pytest.raises(WikiEmbeddingError, match="connection refused"):
|
||||
await service.sync_mapping(mapping)
|
||||
|
||||
assert embedding.calls == 2
|
||||
assert client.knowledge_base_calls == 1
|
||||
next_backoff = await service.sync_mapping(mapping)
|
||||
assert next_backoff["failure_count"] == 2
|
||||
assert next_backoff["retry_after_seconds"] == 120
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_successful_recovery_probe_closes_provider_breaker(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
now = 0.0
|
||||
monkeypatch.setattr(wiki_sync, "monotonic", lambda: now)
|
||||
config = LocalWikiIndexConfig(chunk_min_chars=10, min_similarity=0)
|
||||
store = MemoryLlmWikiIndexStore()
|
||||
embedding = RecoverableProviderEmbedding()
|
||||
client = FakeWeKnora()
|
||||
service = WikiSyncService(
|
||||
config,
|
||||
store,
|
||||
embedding,
|
||||
WikiVectorCache(store),
|
||||
client,
|
||||
)
|
||||
mapping = {"id": "kb", "weknora_id": "remote", "name": "Wiki"}
|
||||
|
||||
with pytest.raises(WikiEmbeddingError):
|
||||
await service.sync_mapping(mapping)
|
||||
now = 60.0
|
||||
embedding.available = True
|
||||
|
||||
result = await service.sync_mapping(mapping)
|
||||
|
||||
assert result["status"] == "completed"
|
||||
assert embedding.calls == 3 # failed page, recovery probe, indexed page
|
||||
assert service._embedding_backoff_result("kb") is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_is_incremental_and_full_scan_deletes_missing_pages() -> None:
|
||||
config = LocalWikiIndexConfig(chunk_min_chars=10, min_similarity=0)
|
||||
@@ -157,6 +392,132 @@ async def test_sync_is_incremental_and_full_scan_deletes_missing_pages() -> None
|
||||
assert next(iter(store.pages.values()))["is_remote_deleted"] is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_persists_mapping_scoped_wiki_edges_for_adjacency() -> None:
|
||||
config = LocalWikiIndexConfig(chunk_min_chars=10, min_similarity=0)
|
||||
store = MemoryLlmWikiIndexStore()
|
||||
client = LinkedFakeWeKnora()
|
||||
mapping = {"id": "kb", "weknora_id": "remote", "name": "Wiki"}
|
||||
|
||||
result = await WikiSyncService(config, store, FakeEmbedding(), WikiVectorCache(store), client).sync_mapping(mapping)
|
||||
|
||||
assert result["graph_edges"] == 1
|
||||
assert result["graph_changed"] is True
|
||||
neighbors = await store.get_adjacent_pages(["kb"], {"kb": ["one"]}, direction="outgoing")
|
||||
assert len(neighbors) == 1
|
||||
assert neighbors[0] == {
|
||||
**neighbors[0],
|
||||
"knowledge_base_mapping_id": "kb",
|
||||
"seed_slug": "one",
|
||||
"neighbor_slug": "two",
|
||||
"direction": "outgoing",
|
||||
"source_slug": "one",
|
||||
"target_slug": "two",
|
||||
}
|
||||
assert neighbors[0]["page"]["title"] == "Two Page"
|
||||
|
||||
# A seed belonging to another mapping cannot traverse this mapping's edge.
|
||||
assert await store.get_adjacent_pages(["other"], {"other": ["one"]}, direction="both") == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_losing_lease_during_graph_fetch_cannot_delete_or_replace_snapshot() -> None:
|
||||
config = LocalWikiIndexConfig(chunk_min_chars=10, min_similarity=0)
|
||||
store = MemoryLlmWikiIndexStore()
|
||||
mapping = {"id": "kb", "weknora_id": "remote", "name": "Wiki"}
|
||||
initial_client = LinkedFakeWeKnora()
|
||||
await WikiSyncService(
|
||||
config,
|
||||
store,
|
||||
FakeEmbedding(),
|
||||
WikiVectorCache(store),
|
||||
initial_client,
|
||||
).sync_mapping(mapping)
|
||||
page_snapshot = {
|
||||
page_id: dict(page)
|
||||
for page_id, page in store.pages.items()
|
||||
}
|
||||
edge_snapshot = {
|
||||
mapping_id: [dict(edge) for edge in edges]
|
||||
for mapping_id, edges in store.edges.items()
|
||||
}
|
||||
|
||||
stealing_client = LeaseStealingGraphWeKnora(store)
|
||||
stealing_client.pages.clear()
|
||||
with pytest.raises(WikiSyncLeaseLost, match="lease was lost"):
|
||||
await WikiSyncService(
|
||||
config,
|
||||
store,
|
||||
FakeEmbedding(),
|
||||
WikiVectorCache(store),
|
||||
stealing_client,
|
||||
).sync_mapping(mapping)
|
||||
|
||||
assert store.pages == page_snapshot
|
||||
assert store.edges == edge_snapshot
|
||||
assert store.states["kb"]["lease_owner"] == "replacement-worker"
|
||||
assert store.states["kb"]["active_sync_id"] == "replacement-sync"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_losing_lease_during_embedding_cannot_replace_page_snapshot() -> None:
|
||||
config = LocalWikiIndexConfig(chunk_min_chars=10, min_similarity=0)
|
||||
store = MemoryLlmWikiIndexStore()
|
||||
client = FakeWeKnora()
|
||||
mapping = {"id": "kb", "weknora_id": "remote", "name": "Wiki"}
|
||||
await WikiSyncService(
|
||||
config,
|
||||
store,
|
||||
FakeEmbedding(),
|
||||
WikiVectorCache(store),
|
||||
client,
|
||||
).sync_mapping(mapping)
|
||||
page_snapshot = {
|
||||
page_id: dict(page)
|
||||
for page_id, page in store.pages.items()
|
||||
}
|
||||
vector_snapshot = {
|
||||
page_id: [dict(vector) for vector in vectors]
|
||||
for page_id, vectors in store.vectors.items()
|
||||
}
|
||||
client.pages["one"]["content"] += " Changed by a stale scan."
|
||||
|
||||
with pytest.raises(WikiSyncLeaseLost, match="lease was lost"):
|
||||
await WikiSyncService(
|
||||
config,
|
||||
store,
|
||||
LeaseStealingEmbedding(store),
|
||||
WikiVectorCache(store),
|
||||
client,
|
||||
).sync_mapping(mapping)
|
||||
|
||||
assert store.pages == page_snapshot
|
||||
assert store.vectors == vector_snapshot
|
||||
assert store.states["kb"]["lease_owner"] == "replacement-worker"
|
||||
assert store.states["kb"]["active_sync_id"] == "replacement-sync"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_page_failure_fencing_propagates_lease_loss_without_wrapping() -> None:
|
||||
config = LocalWikiIndexConfig(chunk_min_chars=10, min_similarity=0)
|
||||
store = MemoryLlmWikiIndexStore()
|
||||
client = FakeWeKnora()
|
||||
mapping = {"id": "kb", "weknora_id": "remote", "name": "Wiki"}
|
||||
|
||||
with pytest.raises(WikiSyncLeaseLost, match="recording a page failure"):
|
||||
await WikiSyncService(
|
||||
config,
|
||||
store,
|
||||
LeaseStealingFailingEmbedding(store),
|
||||
WikiVectorCache(store),
|
||||
client,
|
||||
).sync_mapping(mapping)
|
||||
|
||||
assert store.pages == {}
|
||||
assert store.states["kb"]["lease_owner"] == "replacement-worker"
|
||||
assert store.states["kb"]["active_sync_id"] == "replacement-sync"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_page_failure_keeps_previous_generation_and_marks_partial() -> None:
|
||||
config = LocalWikiIndexConfig(chunk_min_chars=10, min_similarity=0)
|
||||
@@ -199,34 +560,101 @@ async def test_fingerprint_change_automatically_rebuilds_entire_mapping() -> Non
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_auto_sync_scheduler_scans_immediately_then_waits(monkeypatch) -> None:
|
||||
calls: list[str] = []
|
||||
async def test_auto_sync_scheduler_scans_immediately_then_only_when_due(monkeypatch) -> None:
|
||||
now = 0.0
|
||||
refreshes: list[float] = []
|
||||
syncs: list[float] = []
|
||||
sleeps: list[float] = []
|
||||
config = SimpleNamespace(
|
||||
enabled=True,
|
||||
auto_sync=True,
|
||||
sync_interval_seconds=25,
|
||||
)
|
||||
|
||||
async def fake_refresh(_app):
|
||||
refreshes.append(now)
|
||||
return config
|
||||
|
||||
async def fake_sleep(seconds: float) -> None:
|
||||
calls.append(f"sleep:{seconds}")
|
||||
raise asyncio.CancelledError
|
||||
|
||||
async def fake_sync_once(app) -> None:
|
||||
calls.append("sync")
|
||||
if calls.count("sync") > 1:
|
||||
nonlocal now
|
||||
sleeps.append(seconds)
|
||||
if len(syncs) >= 2:
|
||||
raise asyncio.CancelledError
|
||||
now += seconds
|
||||
|
||||
async def fake_sync_once(_app, *, config=None) -> None:
|
||||
syncs.append(now)
|
||||
|
||||
monkeypatch.setattr(llmwiki_index_scheduler, "monotonic", lambda: now)
|
||||
monkeypatch.setattr(llmwiki_index_scheduler.asyncio, "sleep", fake_sleep)
|
||||
monkeypatch.setattr(llmwiki_index_scheduler, "_sync_once", fake_sync_once)
|
||||
monkeypatch.setattr(
|
||||
llmwiki_index_scheduler,
|
||||
"_refresh_local_wiki_config",
|
||||
fake_refresh,
|
||||
)
|
||||
app = SimpleNamespace(
|
||||
state=SimpleNamespace(
|
||||
config=SimpleNamespace(
|
||||
llmwiki=SimpleNamespace(
|
||||
local_wiki_index=SimpleNamespace(sync_interval_seconds=120),
|
||||
)
|
||||
)
|
||||
llmwiki_sync_service=object(),
|
||||
llmwiki_store=object(),
|
||||
)
|
||||
)
|
||||
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await llmwiki_index_scheduler.local_wiki_index_scheduler_loop(app)
|
||||
|
||||
assert calls == ["sync", "sleep:120"]
|
||||
assert refreshes == [0.0, 10.0, 20.0, 30.0]
|
||||
assert syncs == [0.0, 30.0]
|
||||
assert sleeps == [10.0, 10.0, 10.0, 10.0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_auto_sync_scheduler_refreshes_provider_between_syncs(monkeypatch) -> None:
|
||||
now = 0.0
|
||||
seen_providers: list[str] = []
|
||||
syncs: list[str] = []
|
||||
providers = iter(("openai", "local", "local"))
|
||||
|
||||
async def fake_refresh(_app):
|
||||
provider = next(providers)
|
||||
seen_providers.append(provider)
|
||||
_app.state.llmwiki_runtime_signature = provider
|
||||
return SimpleNamespace(
|
||||
enabled=True,
|
||||
auto_sync=True,
|
||||
sync_interval_seconds=120,
|
||||
embedding=SimpleNamespace(provider=provider),
|
||||
)
|
||||
|
||||
async def fake_sync_once(_app, *, config=None) -> None:
|
||||
syncs.append(config.embedding.provider)
|
||||
|
||||
async def fake_sleep(seconds: float) -> None:
|
||||
nonlocal now
|
||||
now += seconds
|
||||
if len(seen_providers) >= 3:
|
||||
raise asyncio.CancelledError
|
||||
|
||||
monkeypatch.setattr(llmwiki_index_scheduler, "monotonic", lambda: now)
|
||||
monkeypatch.setattr(llmwiki_index_scheduler.asyncio, "sleep", fake_sleep)
|
||||
monkeypatch.setattr(llmwiki_index_scheduler, "_sync_once", fake_sync_once)
|
||||
monkeypatch.setattr(
|
||||
llmwiki_index_scheduler,
|
||||
"_refresh_local_wiki_config",
|
||||
fake_refresh,
|
||||
)
|
||||
app = SimpleNamespace(
|
||||
state=SimpleNamespace(
|
||||
llmwiki_sync_service=object(),
|
||||
llmwiki_store=object(),
|
||||
)
|
||||
)
|
||||
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await llmwiki_index_scheduler.local_wiki_index_scheduler_loop(app)
|
||||
|
||||
assert seen_providers == ["openai", "local", "local"]
|
||||
assert syncs == ["openai", "local"]
|
||||
|
||||
|
||||
def test_local_wiki_index_defaults_to_automatic_sync() -> None:
|
||||
|
||||
Reference in new issue
Block a user