sync update from LAN repo (source dhk-1 @ 3b1ec5b4)

This commit is contained in:
gozeros committed 2026-11-05 10:56:34 +08:00
1 parent da6521b6cb
commit 264e2200d6
995 files changed
+148105 -13399

No files matched your search

@@ -9,7 +9,11 @@ import pytest
from app.gateway import llmwiki_index_scheduler
from deerflow.config.llmwiki_config import LocalWikiEmbeddingConfig, LocalWikiIndexConfig
from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient
from deerflow.integrations.weknora.local_index import sync as wiki_sync
from deerflow.integrations.weknora.local_index.embedding import (
StrictWikiEmbeddingClient,
WikiEmbeddingError,
)
from deerflow.integrations.weknora.local_index.normalize import normalize_wiki_page_links
from deerflow.integrations.weknora.local_index.sync import WikiSyncLeaseLost, WikiSyncService
from deerflow.integrations.weknora.local_index.vector_cache import WikiVectorCache
@@ -33,6 +37,32 @@ class FailingEmbedding(FakeEmbedding):
raise RuntimeError("embedding unavailable")
class UnavailableProviderEmbedding(FakeEmbedding):
async def embed_texts(self, texts: list[str]) -> list[list[float]]:
self.calls += 1
raise WikiEmbeddingError(
"connection refused",
category="request",
retryable=True,
)
class RecoverableProviderEmbedding(FakeEmbedding):
def __init__(self) -> None:
super().__init__()
self.available = False
async def embed_texts(self, texts: list[str]) -> list[list[float]]:
self.calls += 1
if not self.available:
raise WikiEmbeddingError(
"connection refused",
category="request",
retryable=True,
)
return [[1.0, 0.0] for _ in texts]
class LeaseStealingEmbedding(FakeEmbedding):
def __init__(self, store: MemoryLlmWikiIndexStore) -> None:
super().__init__()
@@ -83,6 +113,33 @@ class FakeWeKnora:
return dict(self.pages[slug])
class ManyPageWeKnora(FakeWeKnora):
def __init__(self, page_count: int = 50) -> None:
super().__init__()
self.pages = {
f"page-{index}": {
"id": f"p{index}",
"slug": f"page-{index}",
"title": f"Page {index}",
"summary": "A long enough summary for indexing and retrieval.",
"content": "# Intro\nA long enough Wiki paragraph for indexing and semantic retrieval.",
"status": "published",
"version": 1,
}
for index in range(page_count)
}
self.detail_calls = 0
self.knowledge_base_calls = 0
async def get_knowledge_base(self, knowledge_base_id: str):
self.knowledge_base_calls += 1
return await super().get_knowledge_base(knowledge_base_id)
async def get_wiki_page(self, knowledge_base_id: str, slug: str):
self.detail_calls += 1
return await super().get_wiki_page(knowledge_base_id, slug)
class LinkedFakeWeKnora(FakeWeKnora):
def __init__(self) -> None:
super().__init__()
@@ -208,6 +265,107 @@ async def test_omitted_dimensions_are_discovered_from_first_response() -> None:
assert client.dimensions == 2
@pytest.mark.asyncio
async def test_provider_failure_aborts_remaining_pages_and_opens_backoff(
monkeypatch: pytest.MonkeyPatch,
) -> None:
now = 0.0
monkeypatch.setattr(wiki_sync, "monotonic", lambda: now)
config = LocalWikiIndexConfig(chunk_min_chars=10, min_similarity=0)
store = MemoryLlmWikiIndexStore()
embedding = UnavailableProviderEmbedding()
client = ManyPageWeKnora()
service = WikiSyncService(
config,
store,
embedding,
WikiVectorCache(store),
client,
)
mapping = {"id": "kb", "weknora_id": "remote", "name": "Wiki"}
with pytest.raises(WikiEmbeddingError, match="connection refused"):
await service.sync_mapping(mapping)
# The provider client's own retry policy is paid for one page only. The
# remaining 49 pages are not fetched or embedded after a shared provider
# failure.
assert embedding.calls == 1
assert client.detail_calls == 1
assert store.states["kb"]["state"] == "failed"
assert store.states["kb"]["last_sync_result"]["failure_scope"] == "embedding_provider"
assert store.states["kb"]["last_sync_result"]["retry_after_seconds"] == 60
backoff = await service.sync_mapping(mapping)
assert backoff == {
"mapping_id": "kb",
"status": "provider_backoff",
"failure_scope": "embedding_provider",
"failure_count": 1,
"retry_after_seconds": 60,
"error": "WikiEmbeddingError: connection refused",
}
assert embedding.calls == 1
assert client.knowledge_base_calls == 1
manual = await service.sync_mapping(mapping, sync_id="manual-sync")
assert manual["status"] == "failed"
assert manual["sync_id"] == "manual-sync"
assert manual["failure_scope"] == "embedding_provider"
assert manual["failure_count"] == 1
assert manual["retry_after_seconds"] == 60
assert store.states["kb"]["state"] == "failed"
assert store.states["kb"]["remote_page_count"] == 50
assert store.states["kb"]["last_sync_result"] == manual
assert embedding.calls == 1
assert client.knowledge_base_calls == 1
# Once due, a single tiny provider probe fails before any second Wiki
# directory scan and doubles the next breaker window.
now = 60.0
with pytest.raises(WikiEmbeddingError, match="connection refused"):
await service.sync_mapping(mapping)
assert embedding.calls == 2
assert client.knowledge_base_calls == 1
next_backoff = await service.sync_mapping(mapping)
assert next_backoff["failure_count"] == 2
assert next_backoff["retry_after_seconds"] == 120
@pytest.mark.asyncio
async def test_successful_recovery_probe_closes_provider_breaker(
monkeypatch: pytest.MonkeyPatch,
) -> None:
now = 0.0
monkeypatch.setattr(wiki_sync, "monotonic", lambda: now)
config = LocalWikiIndexConfig(chunk_min_chars=10, min_similarity=0)
store = MemoryLlmWikiIndexStore()
embedding = RecoverableProviderEmbedding()
client = FakeWeKnora()
service = WikiSyncService(
config,
store,
embedding,
WikiVectorCache(store),
client,
)
mapping = {"id": "kb", "weknora_id": "remote", "name": "Wiki"}
with pytest.raises(WikiEmbeddingError):
await service.sync_mapping(mapping)
now = 60.0
embedding.available = True
result = await service.sync_mapping(mapping)
assert result["status"] == "completed"
assert embedding.calls == 3 # failed page, recovery probe, indexed page
assert service._embedding_backoff_result("kb") is None
@pytest.mark.asyncio
async def test_sync_is_incremental_and_full_scan_deletes_missing_pages() -> None:
config = LocalWikiIndexConfig(chunk_min_chars=10, min_similarity=0)
@@ -499,10 +657,66 @@ async def test_auto_sync_scheduler_refreshes_provider_between_syncs(monkeypatch)
assert syncs == ["openai", "local"]
def test_local_wiki_index_defaults_to_automatic_sync() -> None:
@pytest.mark.asyncio
async def test_disabled_auto_sync_keeps_scheduler_local_only(monkeypatch) -> None:
now = 0.0
refreshes: list[float] = []
syncs: list[float] = []
remote_maintenance_calls: list[str] = []
config = SimpleNamespace(
enabled=True,
auto_sync=False,
sync_interval_seconds=25,
)
async def fake_refresh(_app):
refreshes.append(now)
return config
async def fake_sleep(seconds: float) -> None:
nonlocal now
now += seconds
if len(refreshes) >= 3:
raise asyncio.CancelledError
async def fake_sync_once(_app, *, config=None) -> None:
syncs.append(now)
async def fake_remote_maintenance(_app) -> None:
remote_maintenance_calls.append("called")
monkeypatch.setattr(llmwiki_index_scheduler, "monotonic", lambda: now)
monkeypatch.setattr(llmwiki_index_scheduler.asyncio, "sleep", fake_sleep)
monkeypatch.setattr(llmwiki_index_scheduler, "_sync_once", fake_sync_once)
monkeypatch.setattr(
llmwiki_index_scheduler,
"_upgrade_wiki_generation_configs_once",
fake_remote_maintenance,
)
monkeypatch.setattr(
llmwiki_index_scheduler,
"_refresh_local_wiki_config",
fake_refresh,
)
app = SimpleNamespace(
state=SimpleNamespace(
llmwiki_sync_service=object(),
llmwiki_store=object(),
)
)
with pytest.raises(asyncio.CancelledError):
await llmwiki_index_scheduler.local_wiki_index_scheduler_loop(app)
assert refreshes == [0.0, 10.0, 20.0]
assert syncs == []
assert remote_maintenance_calls == []
def test_local_wiki_index_defaults_to_manual_sync() -> None:
config = LocalWikiIndexConfig()
assert config.auto_sync is True
assert config.auto_sync is False
assert config.sync_interval_seconds == 30