sync update from LAN repo (source master @ ca48c4ec)

This commit is contained in:
gozeros committed 2026-11-05 10:53:54 +08:00
1 parent 8bc8b0e082
commit ea9dcff9c0
873 files changed
+107866 -14011

No files matched your search

@@ -216,8 +216,9 @@ def test_numbered_citation_prompt_is_classic_only() -> None:
@pytest.mark.asyncio
@pytest.mark.parametrize("assistant_id", [None, "deep-research-planner", "deep-research-collector"])
async def test_smart_scope_binds_this_library_to_authorized_database_metadata(
monkeypatch: pytest.MonkeyPatch,
monkeypatch: pytest.MonkeyPatch, assistant_id: str | None,
) -> None:
from app.gateway import llmwiki_rag
@@ -273,7 +274,7 @@ async def test_smart_scope_binds_this_library_to_authorized_database_metadata(
body=SimpleNamespace(
context={"llmwiki_knowledge_base_ids": ["mapping-news"]},
metadata={},
assistant_id=None,
assistant_id=assistant_id,
),
graph_input={"messages": [HumanMessage(content="这个库讲了什么")]},
)
@@ -604,12 +605,14 @@ async def test_wiki_tool_streams_channel_coverage_and_scan_count(monkeypatch) ->
}
],
pattern=pattern,
channel="local_wiki_mirror",
channel="provider_native_regex",
partial=False,
native_regex_supported=True,
unsupported_knowledge_base_count=0,
coverage=[
{
"knowledge_base": "国内新闻库",
"channel": "local_wiki_mirror",
"channel": "provider_native_regex",
"status": "searched",
}
],
@@ -629,13 +632,15 @@ async def test_wiki_tool_streams_channel_coverage_and_scan_count(monkeypatch) ->
]
completed = events[-1]
assert completed["pattern"] == "新闻|发布会"
assert completed["channel"] == "local_wiki_mirror"
assert completed["channel"] == "provider_native_regex"
assert completed["scanned_pages"] == 17
assert completed["result_count"] == 1
assert completed["native_regex_supported"] is True
assert completed["unsupported_knowledge_base_count"] == 0
assert completed["coverage"] == [
{
"knowledge_base": "国内新闻库",
"channel": "local_wiki_mirror",
"channel": "provider_native_regex",
"status": "searched",
}
]
@@ -1023,74 +1028,50 @@ async def test_search_all_failures_are_errors_and_partial_failure_is_structured(
@pytest.mark.asyncio
async def test_wiki_search_selects_local_or_provider_per_mapping(monkeypatch) -> None:
async def test_wiki_search_uses_native_provider_for_each_mapping_concurrently(
monkeypatch,
) -> None:
_patch_scope(monkeypatch, multi=True)
class LocalIndex:
async def list_index_status(self, mapping_ids: list[str]) -> list[dict]:
assert mapping_ids == ["mapping-1", "mapping-2"]
return [
{
"knowledge_base_mapping_id": "mapping-1",
"state": "idle",
"last_completed_at": "2026-10-14T00:00:00Z",
"last_error": None,
"remote_page_count": 1,
"local_page_count": 1,
"failed_page_count": 0,
"last_sync_result": {"status": "completed"},
},
{
"knowledge_base_mapping_id": "mapping-2",
"state": "partial",
"last_completed_at": "2026-10-14T00:00:00Z",
"last_error": "one page failed",
"remote_page_count": 2,
"local_page_count": 1,
"failed_page_count": 1,
},
]
class LocalIndexMustNotRun:
def __init__(self) -> None:
raise AssertionError("smart Wiki regex must use the native provider API")
async def list_pages(self, mapping_ids: list[str], *, limit: int) -> list[dict]:
assert mapping_ids == ["mapping-1"]
assert limit == 250
class Client:
calls: list[tuple[str, str, int]] = []
active = 0
peak = 0
async def search_wiki_pages(
self,
knowledge_base_id: str,
query: str,
*,
limit: int,
) -> list[dict]:
self.calls.append((knowledge_base_id, query, limit))
self.active += 1
self.peak = max(self.peak, self.active)
await asyncio.sleep(0.01)
self.active -= 1
suffix = knowledge_base_id.rsplit("-", 1)[-1]
return [
{
"knowledge_base_mapping_id": "mapping-1",
"slug": "local-page",
"title": "本地 needle 页面",
"summary": "完整镜像",
"content_md": "本地正文",
"slug": f"news/{suffix}",
"title": f"needle 页面 {suffix}",
"content": "needle 正文",
"status": "published",
}
]
class Client:
calls: list[str] = []
async def list_wiki_pages(self, *args, **kwargs) -> dict:
raise AssertionError("Wiki directory enumeration is forbidden")
async def list_wiki_pages(
self,
knowledge_base_id: str,
*,
page: int,
page_size: int,
) -> dict:
self.calls.append(knowledge_base_id)
assert knowledge_base_id == "remote-2"
return {
"pages": [{"slug": "remote-page", "title": "远端页面"}],
"total_pages": 1,
}
async def get_wiki_page(self, knowledge_base_id: str, slug: str) -> dict:
assert (knowledge_base_id, slug) == ("remote-2", "remote-page")
return {
"slug": slug,
"title": "远端页面",
"content": "远端 needle 正文",
}
async def get_wiki_page(self, *args, **kwargs) -> dict:
raise AssertionError("search must not fetch each page body")
client = Client()
monkeypatch.setattr(tools, "get_local_wiki_index_store", LocalIndex)
monkeypatch.setattr(tools, "get_local_wiki_index_store", LocalIndexMustNotRun)
monkeypatch.setattr(tools, "_client", lambda: client)
payload = json.loads(
await tools._wiki_search_impl(
@@ -1103,87 +1084,148 @@ async def test_wiki_search_selects_local_or_provider_per_mapping(monkeypatch) ->
)
assert payload["ok"] is True
assert {item["knowledge_base"] for item in payload["results"]} == {
"制度库1",
"制度库2",
assert payload["native_regex_supported"] is True
assert payload["result_count"] == 2
assert {call[:2] for call in client.calls} == {
("remote-1", "needle"),
("remote-2", "needle"),
}
assert client.calls == ["remote-2"]
assert payload["channel"] == "mixed"
assert payload["coverage"] == [
{
"knowledge_base": "制度库1",
"channel": "local_wiki_mirror",
"status": "searched",
},
{
"knowledge_base": "制度库2",
"channel": "provider",
"status": "searched",
},
assert client.peak == 2
assert payload["channel"] == "provider_native_regex"
assert [item["status"] for item in payload["coverage"]] == [
"searched",
"searched",
]
@pytest.mark.asyncio
async def test_wiki_search_uses_keyword_index_before_bounded_recent_snapshot(
monkeypatch,
) -> None:
async def test_wiki_search_marks_provider_limit_as_truncated(monkeypatch) -> None:
_patch_scope(monkeypatch)
class LocalIndex:
keyword_calls: list[tuple[list[str], list[str], int]] = []
async def list_index_status(self, mapping_ids: list[str]) -> list[dict]:
return [
{
"knowledge_base_mapping_id": "mapping-1",
"state": "idle",
"last_completed_at": "2026-10-14T00:00:00Z",
"last_error": None,
"remote_page_count": 900,
"local_page_count": 900,
"failed_page_count": 0,
"last_sync_result": {"status": "completed"},
}
]
async def search_pages_by_keywords(
class Client:
async def search_wiki_pages(
self,
mapping_ids: list[str],
tokens: list[str],
knowledge_base_id: str,
query: str,
*,
limit: int,
) -> list[dict]:
self.keyword_calls.append((mapping_ids, tokens, limit))
assert (knowledge_base_id, query, limit) == ("remote-1", "needle", 50)
return [
{
"knowledge_base_mapping_id": "mapping-1",
"slug": "news/old-xinghe",
"title": "较早的国内新闻",
"content_md": "星河工程已经进入联合测试阶段。",
"status": "published",
}
]
async def list_pages(self, mapping_ids: list[str], *, limit: int) -> list[dict]:
return [
{
"knowledge_base_mapping_id": "mapping-1",
"slug": f"recent/{index}",
"title": f"近期页面 {index}",
"content_md": "无关内容",
"status": "published",
"slug": f"news/{index}",
"title": f"needle 页面 {index}",
"content": "needle 正文",
}
for index in range(limit)
]
class ProviderMustNotRun:
def __init__(self) -> None:
raise AssertionError("complete local Wiki mirror should be used")
monkeypatch.setattr(tools, "_client", Client)
payload = json.loads(
await tools._wiki_search_impl(
"needle",
{
"knowledge_retrieval_mode": "smart_reasoning",
"llmwiki_knowledge_base_ids": ["mapping-1"],
},
)
)
local = LocalIndex()
monkeypatch.setattr(tools, "get_local_wiki_index_store", lambda: local)
monkeypatch.setattr(tools, "_client", ProviderMustNotRun)
assert payload["scanned_pages"] == 50
assert payload["scan_truncated"] is True
assert payload["result_count"] == tools._MAX_RESULTS
@pytest.mark.asyncio
async def test_wiki_search_handles_supported_and_unsupported_mappings(
monkeypatch,
) -> None:
_patch_scope(monkeypatch, multi=True)
class Client:
async def search_wiki_pages(
self,
knowledge_base_id: str,
query: str,
*,
limit: int,
) -> list[dict]:
if knowledge_base_id == "remote-2":
raise WeKnoraError("route unavailable", status_code=404)
return [{"slug": "news/one", "title": f"{query} 页面"}]
monkeypatch.setattr(tools, "_client", Client)
payload = json.loads(
await tools._wiki_search_impl(
"needle",
{
"knowledge_retrieval_mode": "smart_reasoning",
"llmwiki_knowledge_base_ids": ["mapping-1", "mapping-2"],
},
)
)
assert payload["ok"] is True
assert payload["native_regex_supported"] is True
assert payload["unsupported_knowledge_base_count"] == 1
assert payload["partial"] is False
assert payload["warnings"] == []
assert [item["status"] for item in payload["coverage"]] == [
"searched",
"unsupported",
]
@pytest.mark.asyncio
async def test_wiki_search_without_native_client_method_is_unsupported(
monkeypatch,
) -> None:
_patch_scope(monkeypatch)
class LegacyClient:
async def list_wiki_pages(self, *args, **kwargs) -> dict:
raise AssertionError("legacy directory fallback must not run")
async def get_wiki_page(self, *args, **kwargs) -> dict:
raise AssertionError("legacy page fallback must not run")
monkeypatch.setattr(tools, "_client", LegacyClient)
payload = json.loads(
await tools._wiki_search_impl(
"needle",
{
"knowledge_retrieval_mode": "smart_reasoning",
"llmwiki_knowledge_base_ids": ["mapping-1"],
},
)
)
assert payload["ok"] is True
assert payload["native_regex_supported"] is False
assert payload["unsupported_knowledge_base_count"] == 1
assert payload["results"] == []
assert payload["warnings"] == []
@pytest.mark.asyncio
@pytest.mark.parametrize("status_code", [404, 405])
async def test_wiki_search_skips_unsupported_native_regex_without_fallback(
monkeypatch,
status_code: int,
) -> None:
_patch_scope(monkeypatch)
class Client:
async def search_wiki_pages(self, *args, **kwargs) -> list[dict]:
raise WeKnoraError("route unavailable", status_code=status_code)
async def list_wiki_pages(self, *args, **kwargs) -> dict:
raise AssertionError("unsupported native search must not enumerate Wiki pages")
async def get_wiki_page(self, *args, **kwargs) -> dict:
raise AssertionError("unsupported native search must not read page bodies")
monkeypatch.setattr(tools, "_client", Client)
payload = json.loads(
await tools._wiki_search_impl(
"星河工程",
@@ -1195,75 +1237,61 @@ async def test_wiki_search_uses_keyword_index_before_bounded_recent_snapshot(
)
assert payload["ok"] is True
assert payload["result_count"] == 1
assert payload["results"][0]["slug"] == "news/old-xinghe"
assert payload["results"][0]["matched_fields"] == ["content"]
assert local.keyword_calls == [
(["mapping-1"], ["星河工程"], tools._MAX_REGEX_CANDIDATES)
]
assert payload["result_count"] == 0
assert payload["native_regex_supported"] is False
assert payload["unsupported_knowledge_base_count"] == 1
assert payload["partial"] is False
assert payload["warnings"] == []
assert payload["coverage"][0]["status"] == "unsupported"
@pytest.mark.asyncio
async def test_zero_page_mirror_probes_live_wiki_and_supports_markdown_shape(
async def test_native_wiki_search_result_can_be_read_with_markdown_shape(
monkeypatch,
) -> None:
_patch_scope(monkeypatch)
class EmptyCompletedMirror:
class EmptyMirror:
async def list_index_status(self, mapping_ids: list[str]) -> list[dict]:
return []
class Client:
search_calls: list[tuple[str, str, int]] = []
async def search_wiki_pages(
self,
knowledge_base_id: str,
query: str,
*,
limit: int,
) -> list[dict]:
self.search_calls.append((knowledge_base_id, query, limit))
return [
{
"knowledge_base_mapping_id": "mapping-1",
"state": "idle",
"last_completed_at": "2026-10-14T00:00:00Z",
"last_error": None,
"remote_page_count": 0,
"local_page_count": 0,
"failed_page_count": 0,
"last_sync_result": {"status": "completed"},
"slug": "news/xinghe",
"title": "国内科技新闻",
"description": "科技报道汇总",
"markdown": "星河工程完成关键节点验收。",
"status": "published",
}
]
class Client:
queries: list[str] = []
async def list_wiki_pages(
self,
knowledge_base_id: str,
*,
page: int,
page_size: int,
query: str = "",
) -> dict:
self.queries.append(query)
return {
"pages": [
{
"slug": "news/xinghe",
"title": "国内科技新闻",
}
],
"total_pages": 1,
}
async def list_wiki_pages(self, *args, **kwargs) -> dict:
raise AssertionError("native search must not enumerate the directory")
async def get_wiki_page(self, knowledge_base_id: str, slug: str) -> dict:
assert (knowledge_base_id, slug) == ("remote-1", "news/xinghe")
return {
"slug": slug,
"title": "国内科技新闻",
"description": "科技报道汇总",
"markdown": "星河工程完成关键节点验收。参见 [[news/验收公告]]。",
"out_links": ["news/承建单位"],
"links": [{"slug": "news/关联政策", "title": "关联政策"}],
"outgoing_links": ["news/旧版关联"],
"status": "published",
}
client = Client()
monkeypatch.setattr(
tools,
"get_local_wiki_index_store",
EmptyCompletedMirror,
)
monkeypatch.setattr(tools, "get_local_wiki_index_store", EmptyMirror)
monkeypatch.setattr(tools, "_client", lambda: client)
payload = json.loads(
@@ -1275,12 +1303,8 @@ async def test_zero_page_mirror_probes_live_wiki_and_supports_markdown_shape(
},
)
)
assert payload["ok"] is True
assert payload["channel"] == "provider"
assert payload["results"][0]["matched_fields"] == ["content"]
assert "星河工程" in client.queries
assert "" in client.queries
assert client.search_calls == [("remote-1", "星河工程", 50)]
read = json.loads(
await tools._wiki_read_impl(
@@ -1293,16 +1317,7 @@ async def test_zero_page_mirror_probes_live_wiki_and_supports_markdown_shape(
)
assert read["results"][0]["content_markdown"] == "星河工程完成关键节点验收。参见 [[news/验收公告]]。"
assert read["results"][0]["summary"] == "科技报道汇总"
assert read["results"][0]["links"] == [
"news/承建单位",
"news/验收公告",
{
"slug": "news/关联政策",
"title": "关联政策",
"direction": "outgoing",
},
"news/旧版关联",
]
assert read["results"][0]["links"] == ["news/承建单位", "news/验收公告"]
@pytest.mark.asyncio
@@ -1452,62 +1467,37 @@ async def test_wiki_read_hides_expansion_links_when_database_setting_is_disabled
@pytest.mark.asyncio
async def test_complete_but_stale_mirror_probes_provider_after_local_zero_match(
async def test_wiki_regex_does_not_use_a_stale_local_mirror(
monkeypatch,
) -> None:
_patch_scope(monkeypatch)
class StaleMirror:
async def list_index_status(self, mapping_ids: list[str]) -> list[dict]:
def __init__(self) -> None:
raise AssertionError("smart Wiki regex must not inspect the local mirror")
class Client:
async def search_wiki_pages(
self,
knowledge_base_id: str,
query: str,
*,
limit: int,
) -> list[dict]:
assert (knowledge_base_id, query, limit) == (
"remote-1",
"星河工程",
50,
)
return [
{
"knowledge_base_mapping_id": "mapping-1",
"state": "idle",
"last_completed_at": "2026-10-14T00:00:00Z",
"last_error": None,
"remote_page_count": 1,
"local_page_count": 1,
"failed_page_count": 0,
"last_sync_result": {"status": "completed"},
}
]
async def search_pages_by_keywords(self, *args, **kwargs) -> list[dict]:
return []
async def list_pages(self, *args, **kwargs) -> list[dict]:
return [
{
"knowledge_base_mapping_id": "mapping-1",
"slug": "news/old",
"title": "旧闻",
"content_md": "镜像中还没有新报道。",
"slug": "news/new",
"title": "新报道",
"content": "星河工程发布最新进展。",
"status": "published",
}
]
class Client:
async def list_wiki_pages(
self,
knowledge_base_id: str,
*,
page: int,
page_size: int,
query: str = "",
) -> dict:
return {
"pages": [{"slug": "news/new", "title": "新报道"}],
"total_pages": 1,
}
async def get_wiki_page(self, knowledge_base_id: str, slug: str) -> dict:
return {
"slug": slug,
"title": "新报道",
"content": "星河工程发布最新进展。",
"status": "published",
}
monkeypatch.setattr(tools, "get_local_wiki_index_store", StaleMirror)
monkeypatch.setattr(tools, "_client", Client)
payload = json.loads(
@@ -1522,7 +1512,7 @@ async def test_complete_but_stale_mirror_probes_provider_after_local_zero_match(
assert payload["result_count"] == 1
assert payload["results"][0]["slug"] == "news/new"
assert payload["channel"] == "local_wiki_mirror_then_provider"
assert payload["channel"] == "provider_native_regex"
@pytest.mark.asyncio
@@ -1531,15 +1521,10 @@ async def test_wiki_provider_all_failures_return_error_and_partial_is_warning(
) -> None:
_patch_scope(monkeypatch, multi=True)
class LocalIndex:
async def list_index_status(self, mapping_ids: list[str]) -> list[dict]:
return []
class AllFailedClient:
async def list_wiki_pages(self, *args, **kwargs) -> dict:
raise WeKnoraError("down")
async def search_wiki_pages(self, *args, **kwargs) -> list[dict]:
raise WeKnoraError("down", status_code=503)
monkeypatch.setattr(tools, "get_local_wiki_index_store", LocalIndex)
monkeypatch.setattr(tools, "_client", AllFailedClient)
failed = json.loads(
await tools._wiki_search_impl(
@@ -1554,22 +1539,16 @@ async def test_wiki_provider_all_failures_return_error_and_partial_is_warning(
assert failed["error"]["code"] == "PROVIDER_UNAVAILABLE"
class PartiallyFailedClient:
async def list_wiki_pages(
async def search_wiki_pages(
self,
knowledge_base_id: str,
query: str,
*,
page: int,
page_size: int,
) -> dict:
limit: int,
) -> list[dict]:
if knowledge_base_id == "remote-2":
raise WeKnoraError("down")
return {
"pages": [{"slug": "ok", "title": "needle"}],
"total_pages": 1,
}
async def get_wiki_page(self, knowledge_base_id: str, slug: str) -> dict:
return {"slug": slug, "title": "needle", "content": "正文"}
raise WeKnoraError("down", status_code=503)
return [{"slug": "ok", "title": "needle", "content": "正文"}]
monkeypatch.setattr(tools, "_client", PartiallyFailedClient)
partial = json.loads(
@@ -1584,7 +1563,7 @@ async def test_wiki_provider_all_failures_return_error_and_partial_is_warning(
assert partial["ok"] is True
assert partial["partial"] is True
assert partial["warnings"] == [
{"code": "PROVIDER_WIKI_LIST_FAILED", "knowledge_base": "制度库2"}
{"code": "PROVIDER_WIKI_SEARCH_FAILED", "knowledge_base": "制度库2"}
]
@@ -1891,3 +1870,34 @@ def test_regex_execution_has_a_hard_timeout() -> None:
with pytest.raises(tools.SmartKnowledgeToolError) as caught:
tools._regex_search(expression, "a" * 10_000 + "!")
assert caught.value.code == "REGEX_TIMEOUT"
@pytest.mark.asyncio
@pytest.mark.parametrize("assistant_id", ["deep-research-planner", "deep-research-collector", "custom-agent"])
async def test_research_smart_scope_cannot_access_unauthorized_library(monkeypatch, assistant_id):
from app.gateway import llmwiki_rag
request = _request(None)
request.app.state.llmwiki_store = _MappingStore()
monkeypatch.setattr(llmwiki_rag, "get_resolved_llmwiki_runtime", lambda _: SimpleNamespace(weknora_enabled=True))
monkeypatch.setattr(llmwiki_rag, "_agent_bound_ids", lambda _: [])
async def actor(_request):
return "user-1", False
monkeypatch.setattr(llmwiki_rag, "_actor", actor)
with pytest.raises(HTTPException) as caught:
await llmwiki_rag.build_llmwiki_smart_reasoning_context(
request=request,
body=SimpleNamespace(context={"llmwiki_knowledge_base_ids": ["not-authorized"]}, metadata={}, assistant_id=assistant_id),
graph_input={"messages": [HumanMessage(content="test")]},
)
assert caught.value.status_code == 404
@pytest.mark.parametrize("agent_id", ["deep-research-planner", "deep-research-collector"])
def test_builtin_research_tools_use_caller_authorized_scope(agent_id):
assert tools._allowed_agent_ids({"agent_id": agent_id}) is None
from deerflow.tools.builtins.llmwiki_search_tool import _agent_allowed_ids
assert _agent_allowed_ids({"agent_id": agent_id}) is None