sync update from LAN repo (source dhk-1 @ 3b1ec5b4)
This commit is contained in:
1 parent
da6521b6cb
commit
264e2200d6
995 files changed
+148105
-13399
No files matched your search
@@ -9,7 +9,11 @@ import pytest
|
||||
|
||||
from app.gateway import llmwiki_index_scheduler
|
||||
from deerflow.config.llmwiki_config import LocalWikiEmbeddingConfig, LocalWikiIndexConfig
|
||||
from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient
|
||||
from deerflow.integrations.weknora.local_index import sync as wiki_sync
|
||||
from deerflow.integrations.weknora.local_index.embedding import (
|
||||
StrictWikiEmbeddingClient,
|
||||
WikiEmbeddingError,
|
||||
)
|
||||
from deerflow.integrations.weknora.local_index.normalize import normalize_wiki_page_links
|
||||
from deerflow.integrations.weknora.local_index.sync import WikiSyncLeaseLost, WikiSyncService
|
||||
from deerflow.integrations.weknora.local_index.vector_cache import WikiVectorCache
|
||||
@@ -33,6 +37,32 @@ class FailingEmbedding(FakeEmbedding):
|
||||
raise RuntimeError("embedding unavailable")
|
||||
|
||||
|
||||
class UnavailableProviderEmbedding(FakeEmbedding):
|
||||
async def embed_texts(self, texts: list[str]) -> list[list[float]]:
|
||||
self.calls += 1
|
||||
raise WikiEmbeddingError(
|
||||
"connection refused",
|
||||
category="request",
|
||||
retryable=True,
|
||||
)
|
||||
|
||||
|
||||
class RecoverableProviderEmbedding(FakeEmbedding):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.available = False
|
||||
|
||||
async def embed_texts(self, texts: list[str]) -> list[list[float]]:
|
||||
self.calls += 1
|
||||
if not self.available:
|
||||
raise WikiEmbeddingError(
|
||||
"connection refused",
|
||||
category="request",
|
||||
retryable=True,
|
||||
)
|
||||
return [[1.0, 0.0] for _ in texts]
|
||||
|
||||
|
||||
class LeaseStealingEmbedding(FakeEmbedding):
|
||||
def __init__(self, store: MemoryLlmWikiIndexStore) -> None:
|
||||
super().__init__()
|
||||
@@ -83,6 +113,33 @@ class FakeWeKnora:
|
||||
return dict(self.pages[slug])
|
||||
|
||||
|
||||
class ManyPageWeKnora(FakeWeKnora):
|
||||
def __init__(self, page_count: int = 50) -> None:
|
||||
super().__init__()
|
||||
self.pages = {
|
||||
f"page-{index}": {
|
||||
"id": f"p{index}",
|
||||
"slug": f"page-{index}",
|
||||
"title": f"Page {index}",
|
||||
"summary": "A long enough summary for indexing and retrieval.",
|
||||
"content": "# Intro\nA long enough Wiki paragraph for indexing and semantic retrieval.",
|
||||
"status": "published",
|
||||
"version": 1,
|
||||
}
|
||||
for index in range(page_count)
|
||||
}
|
||||
self.detail_calls = 0
|
||||
self.knowledge_base_calls = 0
|
||||
|
||||
async def get_knowledge_base(self, knowledge_base_id: str):
|
||||
self.knowledge_base_calls += 1
|
||||
return await super().get_knowledge_base(knowledge_base_id)
|
||||
|
||||
async def get_wiki_page(self, knowledge_base_id: str, slug: str):
|
||||
self.detail_calls += 1
|
||||
return await super().get_wiki_page(knowledge_base_id, slug)
|
||||
|
||||
|
||||
class LinkedFakeWeKnora(FakeWeKnora):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
@@ -208,6 +265,107 @@ async def test_omitted_dimensions_are_discovered_from_first_response() -> None:
|
||||
assert client.dimensions == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_provider_failure_aborts_remaining_pages_and_opens_backoff(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
now = 0.0
|
||||
monkeypatch.setattr(wiki_sync, "monotonic", lambda: now)
|
||||
config = LocalWikiIndexConfig(chunk_min_chars=10, min_similarity=0)
|
||||
store = MemoryLlmWikiIndexStore()
|
||||
embedding = UnavailableProviderEmbedding()
|
||||
client = ManyPageWeKnora()
|
||||
service = WikiSyncService(
|
||||
config,
|
||||
store,
|
||||
embedding,
|
||||
WikiVectorCache(store),
|
||||
client,
|
||||
)
|
||||
mapping = {"id": "kb", "weknora_id": "remote", "name": "Wiki"}
|
||||
|
||||
with pytest.raises(WikiEmbeddingError, match="connection refused"):
|
||||
await service.sync_mapping(mapping)
|
||||
|
||||
# The provider client's own retry policy is paid for one page only. The
|
||||
# remaining 49 pages are not fetched or embedded after a shared provider
|
||||
# failure.
|
||||
assert embedding.calls == 1
|
||||
assert client.detail_calls == 1
|
||||
assert store.states["kb"]["state"] == "failed"
|
||||
assert store.states["kb"]["last_sync_result"]["failure_scope"] == "embedding_provider"
|
||||
assert store.states["kb"]["last_sync_result"]["retry_after_seconds"] == 60
|
||||
|
||||
backoff = await service.sync_mapping(mapping)
|
||||
|
||||
assert backoff == {
|
||||
"mapping_id": "kb",
|
||||
"status": "provider_backoff",
|
||||
"failure_scope": "embedding_provider",
|
||||
"failure_count": 1,
|
||||
"retry_after_seconds": 60,
|
||||
"error": "WikiEmbeddingError: connection refused",
|
||||
}
|
||||
assert embedding.calls == 1
|
||||
assert client.knowledge_base_calls == 1
|
||||
|
||||
manual = await service.sync_mapping(mapping, sync_id="manual-sync")
|
||||
|
||||
assert manual["status"] == "failed"
|
||||
assert manual["sync_id"] == "manual-sync"
|
||||
assert manual["failure_scope"] == "embedding_provider"
|
||||
assert manual["failure_count"] == 1
|
||||
assert manual["retry_after_seconds"] == 60
|
||||
assert store.states["kb"]["state"] == "failed"
|
||||
assert store.states["kb"]["remote_page_count"] == 50
|
||||
assert store.states["kb"]["last_sync_result"] == manual
|
||||
assert embedding.calls == 1
|
||||
assert client.knowledge_base_calls == 1
|
||||
|
||||
# Once due, a single tiny provider probe fails before any second Wiki
|
||||
# directory scan and doubles the next breaker window.
|
||||
now = 60.0
|
||||
with pytest.raises(WikiEmbeddingError, match="connection refused"):
|
||||
await service.sync_mapping(mapping)
|
||||
|
||||
assert embedding.calls == 2
|
||||
assert client.knowledge_base_calls == 1
|
||||
next_backoff = await service.sync_mapping(mapping)
|
||||
assert next_backoff["failure_count"] == 2
|
||||
assert next_backoff["retry_after_seconds"] == 120
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_successful_recovery_probe_closes_provider_breaker(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
now = 0.0
|
||||
monkeypatch.setattr(wiki_sync, "monotonic", lambda: now)
|
||||
config = LocalWikiIndexConfig(chunk_min_chars=10, min_similarity=0)
|
||||
store = MemoryLlmWikiIndexStore()
|
||||
embedding = RecoverableProviderEmbedding()
|
||||
client = FakeWeKnora()
|
||||
service = WikiSyncService(
|
||||
config,
|
||||
store,
|
||||
embedding,
|
||||
WikiVectorCache(store),
|
||||
client,
|
||||
)
|
||||
mapping = {"id": "kb", "weknora_id": "remote", "name": "Wiki"}
|
||||
|
||||
with pytest.raises(WikiEmbeddingError):
|
||||
await service.sync_mapping(mapping)
|
||||
now = 60.0
|
||||
embedding.available = True
|
||||
|
||||
result = await service.sync_mapping(mapping)
|
||||
|
||||
assert result["status"] == "completed"
|
||||
assert embedding.calls == 3 # failed page, recovery probe, indexed page
|
||||
assert service._embedding_backoff_result("kb") is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_is_incremental_and_full_scan_deletes_missing_pages() -> None:
|
||||
config = LocalWikiIndexConfig(chunk_min_chars=10, min_similarity=0)
|
||||
@@ -499,10 +657,66 @@ async def test_auto_sync_scheduler_refreshes_provider_between_syncs(monkeypatch)
|
||||
assert syncs == ["openai", "local"]
|
||||
|
||||
|
||||
def test_local_wiki_index_defaults_to_automatic_sync() -> None:
|
||||
@pytest.mark.asyncio
|
||||
async def test_disabled_auto_sync_keeps_scheduler_local_only(monkeypatch) -> None:
|
||||
now = 0.0
|
||||
refreshes: list[float] = []
|
||||
syncs: list[float] = []
|
||||
remote_maintenance_calls: list[str] = []
|
||||
config = SimpleNamespace(
|
||||
enabled=True,
|
||||
auto_sync=False,
|
||||
sync_interval_seconds=25,
|
||||
)
|
||||
|
||||
async def fake_refresh(_app):
|
||||
refreshes.append(now)
|
||||
return config
|
||||
|
||||
async def fake_sleep(seconds: float) -> None:
|
||||
nonlocal now
|
||||
now += seconds
|
||||
if len(refreshes) >= 3:
|
||||
raise asyncio.CancelledError
|
||||
|
||||
async def fake_sync_once(_app, *, config=None) -> None:
|
||||
syncs.append(now)
|
||||
|
||||
async def fake_remote_maintenance(_app) -> None:
|
||||
remote_maintenance_calls.append("called")
|
||||
|
||||
monkeypatch.setattr(llmwiki_index_scheduler, "monotonic", lambda: now)
|
||||
monkeypatch.setattr(llmwiki_index_scheduler.asyncio, "sleep", fake_sleep)
|
||||
monkeypatch.setattr(llmwiki_index_scheduler, "_sync_once", fake_sync_once)
|
||||
monkeypatch.setattr(
|
||||
llmwiki_index_scheduler,
|
||||
"_upgrade_wiki_generation_configs_once",
|
||||
fake_remote_maintenance,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
llmwiki_index_scheduler,
|
||||
"_refresh_local_wiki_config",
|
||||
fake_refresh,
|
||||
)
|
||||
app = SimpleNamespace(
|
||||
state=SimpleNamespace(
|
||||
llmwiki_sync_service=object(),
|
||||
llmwiki_store=object(),
|
||||
)
|
||||
)
|
||||
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await llmwiki_index_scheduler.local_wiki_index_scheduler_loop(app)
|
||||
|
||||
assert refreshes == [0.0, 10.0, 20.0]
|
||||
assert syncs == []
|
||||
assert remote_maintenance_calls == []
|
||||
|
||||
|
||||
def test_local_wiki_index_defaults_to_manual_sync() -> None:
|
||||
config = LocalWikiIndexConfig()
|
||||
|
||||
assert config.auto_sync is True
|
||||
assert config.auto_sync is False
|
||||
assert config.sync_interval_seconds == 30
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user