feat: integrate WeKnora llmwiki knowledge flow
This commit is contained in:
1 parent
a0a75a9964
commit
e0c09f6704
63 files changed
+7045
-197
No files matched your search
@@ -0,0 +1,100 @@
|
||||
"""Regression coverage for the unauthenticated Gateway health probes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.gateway import auth_middleware
|
||||
from app.gateway.app import app as gateway_app
|
||||
from app.gateway.auth_middleware import PUBLIC_HEALTH_PATHS, AuthMiddleware
|
||||
|
||||
|
||||
def _health_only_app() -> FastAPI:
|
||||
app = FastAPI()
|
||||
|
||||
async def health_check() -> dict[str, str]:
|
||||
return {"status": "healthy", "service": "deer-flow-gateway"}
|
||||
|
||||
for path in PUBLIC_HEALTH_PATHS:
|
||||
app.add_api_route(path, health_check, methods=["GET", "HEAD"])
|
||||
app.add_middleware(AuthMiddleware)
|
||||
return app
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", sorted(PUBLIC_HEALTH_PATHS))
|
||||
@pytest.mark.parametrize("method", ["GET", "HEAD"])
|
||||
def test_health_probe_is_public_when_auth_is_enabled(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
path: str,
|
||||
method: str,
|
||||
) -> None:
|
||||
monkeypatch.setattr(auth_middleware, "is_auth_disabled", lambda: False)
|
||||
|
||||
with TestClient(_health_only_app()) as client:
|
||||
client.cookies.set("access_token", "invalid-token")
|
||||
response = client.request(
|
||||
method,
|
||||
path,
|
||||
headers={"Authorization": "Bearer invalid-token"},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
if method == "GET":
|
||||
assert response.json() == {"status": "healthy", "service": "deer-flow-gateway"}
|
||||
else:
|
||||
assert response.content == b""
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", sorted(PUBLIC_HEALTH_PATHS))
|
||||
def test_health_probe_allows_a_trailing_slash(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
path: str,
|
||||
) -> None:
|
||||
monkeypatch.setattr(auth_middleware, "is_auth_disabled", lambda: False)
|
||||
|
||||
with TestClient(_health_only_app(), follow_redirects=False) as client:
|
||||
response = client.get(f"{path}/")
|
||||
|
||||
# FastAPI redirects to the canonical route; importantly, auth must not turn
|
||||
# the request into a 401 before routing can perform that redirect.
|
||||
assert response.status_code == 307
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", sorted(PUBLIC_HEALTH_PATHS))
|
||||
def test_health_probe_is_public_behind_a_reverse_proxy_prefix(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
path: str,
|
||||
) -> None:
|
||||
monkeypatch.setattr(auth_middleware, "is_auth_disabled", lambda: False)
|
||||
|
||||
with TestClient(_health_only_app(), root_path="/deerflow") as client:
|
||||
response = client.get(f"/deerflow{path}")
|
||||
|
||||
assert response.status_code == 200
|
||||
|
||||
|
||||
def test_gateway_registers_every_public_health_path_for_get_and_head() -> None:
|
||||
methods_by_path = {
|
||||
route.path: route.methods
|
||||
for route in gateway_app.routes
|
||||
if hasattr(route, "methods")
|
||||
}
|
||||
|
||||
for path in PUBLIC_HEALTH_PATHS:
|
||||
assert {"GET", "HEAD"} <= methods_by_path[path]
|
||||
|
||||
|
||||
def test_non_health_route_still_requires_auth(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(auth_middleware, "is_auth_disabled", lambda: False)
|
||||
app = _health_only_app()
|
||||
|
||||
@app.get("/private")
|
||||
async def private() -> dict[str, bool]:
|
||||
return {"ok": True}
|
||||
|
||||
with TestClient(app) as client:
|
||||
response = client.get("/private")
|
||||
|
||||
assert response.status_code == 401
|
||||
@@ -0,0 +1,418 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
from starlette.datastructures import Headers, QueryParams
|
||||
|
||||
from deerflow.config.agents_config import AgentConfig
|
||||
from deerflow.config.llmwiki_config import (
|
||||
LlmWikiConfig,
|
||||
LlmWikiRuntimeOverride,
|
||||
resolve_llmwiki_runtime,
|
||||
)
|
||||
from deerflow.integrations.weknora.client import WeKnoraClient, WeKnoraError
|
||||
from deerflow.persistence.llmwiki import MemoryLlmWikiStore
|
||||
from deerflow.tools.builtins.llmwiki_search_tool import selected_knowledge_base_ids
|
||||
|
||||
|
||||
def test_empty_weknora_address_keeps_legacy_mode() -> None:
|
||||
runtime = resolve_llmwiki_runtime(LlmWikiConfig())
|
||||
|
||||
assert runtime.provider == "legacy"
|
||||
assert runtime.api_base_url == ""
|
||||
|
||||
|
||||
def test_configured_weknora_address_enables_weknora_mode() -> None:
|
||||
config = LlmWikiConfig.model_validate(
|
||||
{
|
||||
"weknora": {
|
||||
"api_base_url": "http://weknora-app:8080/",
|
||||
"web_base_url": "https://weknora.example.test/",
|
||||
"admin_email": "admin@example.test",
|
||||
"admin_password": "secret",
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
runtime = resolve_llmwiki_runtime(config)
|
||||
|
||||
assert runtime.provider == "weknora"
|
||||
assert runtime.api_base_url == "http://weknora-app:8080"
|
||||
assert runtime.web_base_url == "https://weknora.example.test"
|
||||
assert runtime.admin_email == "admin@example.test"
|
||||
assert runtime.admin_password == "secret"
|
||||
|
||||
|
||||
def test_runtime_override_can_force_legacy_or_reset_to_file() -> None:
|
||||
config = LlmWikiConfig.model_validate({"weknora": {"api_base_url": "http://weknora:8080"}})
|
||||
|
||||
forced_legacy = resolve_llmwiki_runtime(
|
||||
config,
|
||||
LlmWikiRuntimeOverride(enabled=True, api_base_url=""),
|
||||
)
|
||||
reset_to_file = resolve_llmwiki_runtime(
|
||||
config,
|
||||
LlmWikiRuntimeOverride(enabled=False, api_base_url=""),
|
||||
)
|
||||
|
||||
assert forced_legacy.provider == "legacy"
|
||||
assert reset_to_file.provider == "weknora"
|
||||
|
||||
|
||||
def test_weknora_url_rejects_credentials_and_non_http_schemes() -> None:
|
||||
with pytest.raises(ValueError):
|
||||
LlmWikiConfig.model_validate({"weknora": {"api_base_url": "ftp://weknora.local"}})
|
||||
with pytest.raises(ValueError):
|
||||
LlmWikiConfig.model_validate({"weknora": {"api_base_url": "http://user:pass@weknora.local"}})
|
||||
|
||||
|
||||
def test_agent_config_tracks_allowed_llmwiki_knowledge_bases() -> None:
|
||||
config = AgentConfig(
|
||||
name="Research agent",
|
||||
llmwiki_knowledge_base_ids=["kb-one", "kb-two", "kb-one"],
|
||||
)
|
||||
|
||||
assert config.llmwiki_knowledge_base_ids == ["kb-one", "kb-two"]
|
||||
|
||||
|
||||
def test_conversation_selection_is_deduplicated_and_absence_is_distinct() -> None:
|
||||
assert selected_knowledge_base_ids({}) is None
|
||||
assert selected_knowledge_base_ids({"llmwiki_knowledge_base_ids": []}) == []
|
||||
assert selected_knowledge_base_ids({"llmwiki_knowledge_base_ids": ["kb-one", "kb-one", "", "kb-two"]}) == ["kb-one", "kb-two"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_weknora_client_sends_no_credentials_and_normalizes_search() -> None:
|
||||
requests: list[httpx.Request] = []
|
||||
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
requests.append(request)
|
||||
assert "X-API-Key" not in request.headers
|
||||
assert "Authorization" not in request.headers
|
||||
assert json.loads(request.content) == {
|
||||
"query": "deployment guide",
|
||||
"knowledge_base_ids": ["kb-one", "kb-two"],
|
||||
}
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"success": True,
|
||||
"data": [
|
||||
{
|
||||
"id": "chunk-one",
|
||||
"content": "matched passage",
|
||||
"knowledge_id": "doc-one",
|
||||
"knowledge_base_id": "kb-one",
|
||||
"knowledge_title": "Guide",
|
||||
"score": 0.93,
|
||||
}
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http_client:
|
||||
client = WeKnoraClient(
|
||||
base_url="http://weknora.local:8080",
|
||||
http_client=http_client,
|
||||
)
|
||||
results = await client.search("deployment guide", ["kb-one", "kb-two"])
|
||||
|
||||
assert requests[0].url.path == "/api/v1/knowledge-search"
|
||||
assert results[0]["chunk_id"] == "chunk-one"
|
||||
assert results[0]["title"] == "Guide"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_weknora_client_resolves_default_embedding_model_inside_weknora() -> None:
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
if request.method == "GET" and request.url.path == "/api/v1/models":
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"success": True,
|
||||
"data": [
|
||||
{"id": "embed-default", "type": "Embedding", "is_default": True},
|
||||
],
|
||||
},
|
||||
)
|
||||
assert request.method == "POST"
|
||||
assert request.url.path == "/api/v1/knowledge-bases"
|
||||
assert json.loads(request.content)["embedding_model_id"] == "embed-default"
|
||||
return httpx.Response(201, json={"success": True, "data": {"id": "kb-created", "name": "Docs"}})
|
||||
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http_client:
|
||||
client = WeKnoraClient(
|
||||
base_url="http://weknora.local:8080",
|
||||
http_client=http_client,
|
||||
)
|
||||
created = await client.create_knowledge_base(name="Docs")
|
||||
|
||||
assert created["id"] == "kb-created"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_weknora_client_creates_wiki_with_weknora_owned_models() -> None:
|
||||
model_reads = 0
|
||||
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
nonlocal model_reads
|
||||
if request.method == "GET" and request.url.path == "/api/v1/models":
|
||||
model_reads += 1
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"data": [
|
||||
{"id": "embed-one", "type": "Embedding", "is_default": True},
|
||||
{"id": "qa-one", "type": "KnowledgeQA", "status": "active"},
|
||||
]
|
||||
},
|
||||
)
|
||||
payload = json.loads(request.content)
|
||||
assert payload["embedding_model_id"] == "embed-one"
|
||||
assert payload["summary_model_id"] == "qa-one"
|
||||
assert payload["indexing_strategy"] == {
|
||||
"vector_enabled": True,
|
||||
"keyword_enabled": True,
|
||||
"wiki_enabled": True,
|
||||
"graph_enabled": False,
|
||||
}
|
||||
assert payload["wiki_config"]["synthesis_model_id"] == "qa-one"
|
||||
return httpx.Response(201, json={"data": {"id": "wiki-one", "name": "Wiki"}})
|
||||
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http_client:
|
||||
client = WeKnoraClient(base_url="http://weknora.local:8080", http_client=http_client)
|
||||
created = await client.create_knowledge_base(name="Wiki", wiki_enabled=True)
|
||||
|
||||
assert created["id"] == "wiki-one"
|
||||
assert model_reads == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_weknora_client_imports_url_manual_content_and_lists_wiki_pages() -> None:
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.path.endswith("/knowledge/url"):
|
||||
assert json.loads(request.content) == {"url": "https://example.test/guide"}
|
||||
return httpx.Response(201, json={"data": {"id": "url-one"}})
|
||||
if request.url.path.endswith("/knowledge/manual"):
|
||||
assert json.loads(request.content) == {
|
||||
"title": "Guide",
|
||||
"content": "# Guide",
|
||||
"status": "publish",
|
||||
}
|
||||
return httpx.Response(201, json={"data": {"id": "manual-one"}})
|
||||
assert request.url.path == "/api/v1/knowledgebase/kb-one/wiki/pages"
|
||||
assert request.url.params["query"] == "guide"
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={"pages": [{"slug": "guide", "title": "Guide"}], "total": 1, "page": 1, "page_size": 50},
|
||||
)
|
||||
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http_client:
|
||||
client = WeKnoraClient(base_url="http://weknora.local:8080", http_client=http_client)
|
||||
await client.import_document_url("kb-one", url="https://example.test/guide")
|
||||
await client.create_manual_document("kb-one", title="Guide", content="# Guide")
|
||||
pages = await client.list_wiki_pages("kb-one", query="guide")
|
||||
|
||||
assert pages["total"] == 1
|
||||
assert pages["pages"][0]["slug"] == "guide"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_weknora_client_reads_document_chunks_and_wiki_graph() -> None:
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.path == "/api/v1/chunks/doc-one":
|
||||
assert request.url.params["page"] == "2"
|
||||
assert request.url.params["page_size"] == "10"
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"success": True,
|
||||
"data": [{"id": "chunk-one", "content": "parsed content", "chunk_index": 10}],
|
||||
"total": 31,
|
||||
"page": 2,
|
||||
"page_size": 10,
|
||||
},
|
||||
)
|
||||
assert request.url.path == "/api/v1/knowledgebase/kb-one/wiki/graph"
|
||||
assert request.url.params["mode"] == "overview"
|
||||
assert request.url.params["limit"] == "50"
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"success": True,
|
||||
"data": {
|
||||
"nodes": [{"slug": "entity/one", "title": "Entity One", "page_type": "entity"}],
|
||||
"edges": [],
|
||||
"meta": {"mode": "overview", "total": 1, "returned": 1},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http_client:
|
||||
client = WeKnoraClient(base_url="http://weknora.local:8080", http_client=http_client)
|
||||
chunks = await client.list_chunks("doc-one", page=2, page_size=10)
|
||||
graph = await client.get_wiki_graph("kb-one", limit=50)
|
||||
|
||||
assert chunks["total"] == 31
|
||||
assert chunks["items"][0]["content"] == "parsed content"
|
||||
assert graph["nodes"][0]["slug"] == "entity/one"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_weknora_client_proxies_original_document_preview_without_credentials() -> None:
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
assert request.url.path == "/api/v1/knowledge/doc-one/preview"
|
||||
assert "X-API-Key" not in request.headers
|
||||
assert "Authorization" not in request.headers
|
||||
return httpx.Response(200, content=b"preview", headers={"Content-Type": "text/plain; charset=utf-8"})
|
||||
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http_client:
|
||||
client = WeKnoraClient(base_url="http://weknora.local:8080", http_client=http_client)
|
||||
content, media_type = await client.get_document_preview("doc-one")
|
||||
|
||||
assert content == b"preview"
|
||||
assert media_type == "text/plain"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_memory_store_enforces_owner_and_public_visibility() -> None:
|
||||
store = MemoryLlmWikiStore()
|
||||
created = await store.create_mapping(
|
||||
weknora_id="wk-private",
|
||||
owner_user_id="user-a",
|
||||
name="Private notes",
|
||||
description="",
|
||||
kb_type="document",
|
||||
)
|
||||
|
||||
assert [row["id"] for row in await store.list_visible("user-a", scope="personal")] == [created["id"]]
|
||||
assert await store.list_visible("user-b", scope="personal") == []
|
||||
assert await store.get_authorized(created["id"], "user-b", write=False, is_admin=False) is None
|
||||
|
||||
published = await store.request_publish(created["id"], "user-a", is_admin=False)
|
||||
assert published and published["publication_status"] == "published"
|
||||
assert [row["id"] for row in await store.list_visible("user-b", scope="public")] == [created["id"]]
|
||||
assert await store.get_authorized(created["id"], "user-b", write=False, is_admin=False) is not None
|
||||
assert await store.get_authorized(created["id"], "user-b", write=True, is_admin=False) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_cleans_owner_mapping_when_remote_is_already_missing(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from app.gateway.routers import llmwiki as llmwiki_router
|
||||
|
||||
deleted: list[str] = []
|
||||
|
||||
class MissingClient:
|
||||
async def delete_knowledge_base(self, _: str) -> None:
|
||||
raise WeKnoraError("WeKnora resource was not found", status_code=404)
|
||||
|
||||
class Store:
|
||||
async def delete_mapping(self, mapping_id: str) -> None:
|
||||
deleted.append(mapping_id)
|
||||
|
||||
async def authorized_mapping(*_: object, **__: object) -> tuple[dict[str, str], str, bool]:
|
||||
return {"weknora_id": "remote-missing"}, "owner", False
|
||||
|
||||
monkeypatch.setattr(llmwiki_router, "_authorized_mapping", authorized_mapping)
|
||||
monkeypatch.setattr(llmwiki_router, "_client_or_503", lambda: MissingClient())
|
||||
monkeypatch.setattr(llmwiki_router, "_store", lambda _: Store())
|
||||
|
||||
await llmwiki_router.delete_knowledge_base(object(), "mapping-one")
|
||||
|
||||
assert deleted == ["mapping-one"]
|
||||
|
||||
|
||||
def test_weknora_iframe_cookie_is_signed_and_short_lived() -> None:
|
||||
from app.gateway.routers import llmwiki as llmwiki_router
|
||||
|
||||
raw = llmwiki_router._sign_embed_payload(
|
||||
{
|
||||
"mapping_id": "local-kb",
|
||||
"weknora_id": "remote-kb",
|
||||
"user_id": "user-a",
|
||||
"is_admin": False,
|
||||
"can_write": True,
|
||||
"exp": 4_102_444_800,
|
||||
}
|
||||
)
|
||||
|
||||
ctx = llmwiki_router._verify_embed_cookie(raw)
|
||||
|
||||
assert ctx.mapping_id == "local-kb"
|
||||
assert ctx.weknora_id == "remote-kb"
|
||||
assert ctx.can_write is True
|
||||
|
||||
with pytest.raises(HTTPException):
|
||||
llmwiki_router._verify_embed_cookie(raw + "tampered")
|
||||
|
||||
|
||||
def test_weknora_iframe_filters_kb_list_to_current_mapping() -> None:
|
||||
from app.gateway.routers import llmwiki as llmwiki_router
|
||||
|
||||
ctx = llmwiki_router.WeKnoraEmbedContext(
|
||||
mapping_id="local-kb",
|
||||
weknora_id="remote-kb",
|
||||
user_id="user-a",
|
||||
is_admin=False,
|
||||
can_write=False,
|
||||
exp=4_102_444_800,
|
||||
)
|
||||
response = httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"success": True,
|
||||
"data": [
|
||||
{"id": "remote-kb", "name": "Allowed"},
|
||||
{"id": "other-kb", "name": "Blocked"},
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
filtered = llmwiki_router._filter_knowledge_base_list_response(response, ctx)
|
||||
|
||||
assert filtered is not None
|
||||
payload = json.loads(filtered)
|
||||
assert payload["data"] == [{"id": "remote-kb", "name": "Allowed"}]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_weknora_iframe_proxy_blocks_other_knowledge_bases() -> None:
|
||||
from app.gateway.routers import llmwiki as llmwiki_router
|
||||
|
||||
class FakeRequest:
|
||||
method = "GET"
|
||||
headers = Headers({})
|
||||
query_params = QueryParams("")
|
||||
|
||||
ctx = llmwiki_router.WeKnoraEmbedContext(
|
||||
mapping_id="local-kb",
|
||||
weknora_id="remote-kb",
|
||||
user_id="user-a",
|
||||
is_admin=False,
|
||||
can_write=False,
|
||||
exp=4_102_444_800,
|
||||
)
|
||||
|
||||
await llmwiki_router._authorize_weknora_api_proxy(
|
||||
FakeRequest(),
|
||||
ctx,
|
||||
object(),
|
||||
"/api/v1/knowledge-bases/remote-kb",
|
||||
b"",
|
||||
"token",
|
||||
)
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await llmwiki_router._authorize_weknora_api_proxy(
|
||||
FakeRequest(),
|
||||
ctx,
|
||||
object(),
|
||||
"/api/v1/knowledge-bases/other-kb",
|
||||
b"",
|
||||
"token",
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 403
|
||||
Reference in new issue
Block a user