Merge branch 'master' of http://192.168.0.200/knowledge-base-bj/deerflow-code into master-nj
This commit is contained in:
commit
5c0864bf0a
40 files changed
+1200
-179
No files matched your search
@@ -0,0 +1,43 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.dialects import mysql
|
||||
|
||||
from app.gateway.routers.admin_users import _apply_run_scope, _apply_tool_scope
|
||||
from deerflow.persistence.run.model import RunRow
|
||||
from deerflow.persistence.tool_metrics.model import ToolCallMetricRow
|
||||
|
||||
|
||||
def _mysql_sql(stmt) -> str:
|
||||
return str(stmt.compile(dialect=mysql.dialect(), compile_kwargs={"literal_binds": True})).upper()
|
||||
|
||||
|
||||
def test_excluding_scheduled_runs_uses_index_friendly_not_exists() -> None:
|
||||
stmt = _apply_run_scope(
|
||||
select(RunRow.run_id),
|
||||
user_column=RunRow.user_id,
|
||||
run_id_column=RunRow.run_id,
|
||||
exclude_user_ids=set(),
|
||||
include_admins=True,
|
||||
include_failed=True,
|
||||
include_scheduled=False,
|
||||
)
|
||||
sql = _mysql_sql(stmt)
|
||||
|
||||
assert "NOT (EXISTS" in sql
|
||||
assert "SCHEDULED_TASK_RUNS" in sql
|
||||
assert " NOT IN " not in sql
|
||||
|
||||
|
||||
def test_tool_aggregation_does_not_join_the_full_runs_table() -> None:
|
||||
stmt = _apply_tool_scope(
|
||||
select(ToolCallMetricRow.tool_name),
|
||||
exclude_user_ids=set(),
|
||||
include_admins=True,
|
||||
include_scheduled=False,
|
||||
)
|
||||
sql = _mysql_sql(stmt)
|
||||
|
||||
assert sql.count("NOT (EXISTS") == 2
|
||||
assert "LEFT OUTER JOIN" not in sql
|
||||
assert " NOT IN " not in sql
|
||||
@@ -40,21 +40,48 @@ async def test_sql_store_atomically_replaces_vectors_and_marks_missing_deleted(t
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
store = SqlLlmWikiIndexStore(session_factory)
|
||||
store = SqlLlmWikiIndexStore(session_factory, write_batch_size=1)
|
||||
assert await store.try_acquire_sync_lease("mapping-1", owner="worker", sync_id="sync-1", lease_seconds=60, fingerprint="fp")
|
||||
assert not await store.try_acquire_sync_lease("mapping-1", owner="worker", sync_id="sync-2", lease_seconds=60, fingerprint="fp")
|
||||
page = normalize_wiki_page("mapping-1", {"id": "p1", "slug": "people/alice", "title": "Alice", "content": "Alice founded Acme.", "status": "published"})
|
||||
blob, dimensions = encode_vector([3.0, 4.0], dimensions=2)
|
||||
saved = await store.replace_page_vectors(
|
||||
page=page,
|
||||
vectors=[{"section_index": 0, "heading": "History", "section_content": "Alice founded Acme.", "content_hash": "section", "embedding_dimensions": dimensions, "vector_blob": blob}],
|
||||
vectors=[
|
||||
{
|
||||
"section_index": index,
|
||||
"heading": "History",
|
||||
"section_content": f"Alice founded Acme. Part {index}",
|
||||
"content_hash": f"section-{index}",
|
||||
"embedding_dimensions": dimensions,
|
||||
"vector_blob": blob,
|
||||
}
|
||||
for index in range(3)
|
||||
],
|
||||
fingerprint="fp",
|
||||
sync_id="sync-1",
|
||||
)
|
||||
snapshot = await store.load_vector_snapshot("mapping-1", "fp")
|
||||
assert saved["slug"] == "people/alice"
|
||||
assert len(snapshot["rows"]) == 1
|
||||
assert store.write_batch_size == 1
|
||||
assert len(snapshot["rows"]) == 3
|
||||
assert decode_vector(snapshot["rows"][0]["vector_blob"], 2).tolist() == pytest.approx([0.6, 0.8])
|
||||
page_statuses = await store.list_page_index_status("mapping-1")
|
||||
assert len(page_statuses) == 1
|
||||
assert page_statuses[0] == {
|
||||
**page_statuses[0],
|
||||
"id": saved["id"],
|
||||
"slug": "people/alice",
|
||||
"title": "Alice",
|
||||
"page_type": "article",
|
||||
"status": "published",
|
||||
"index_status": "ready",
|
||||
"vector_count": 3,
|
||||
"last_error": None,
|
||||
"last_seen_sync_id": "sync-1",
|
||||
}
|
||||
assert page_statuses[0]["indexed_at"]
|
||||
assert page_statuses[0]["updated_at"]
|
||||
|
||||
assert await store.mark_missing_pages_deleted("mapping-1", "sync-2") == 1
|
||||
assert (await store.load_vector_snapshot("mapping-1", "fp"))["rows"] == []
|
||||
|
||||
@@ -690,6 +690,24 @@ async def test_weknora_client_proxies_original_document_preview_without_credenti
|
||||
assert media_type == "text/plain"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_weknora_client_fetches_protected_wiki_image() -> None:
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
assert request.url.path == "/api/v1/knowledge-bases/kb-one/files"
|
||||
assert request.url.params["file_path"] == "resource://wiki/figure.png"
|
||||
return httpx.Response(200, content=b"png-bytes", headers={"Content-Type": "image/png"})
|
||||
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http_client:
|
||||
client = WeKnoraClient(base_url="http://weknora.local:8080", http_client=http_client)
|
||||
content, media_type = await client.get_knowledge_base_file(
|
||||
"kb-one",
|
||||
"resource://wiki/figure.png",
|
||||
)
|
||||
|
||||
assert content == b"png-bytes"
|
||||
assert media_type == "image/png"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_memory_store_enforces_owner_and_public_visibility() -> None:
|
||||
store = MemoryLlmWikiStore()
|
||||
@@ -1358,6 +1376,10 @@ def test_weknora_iframe_html_rewrites_public_prefix_paths() -> None:
|
||||
assert '"allowedWeKnoraIds":["remote-kb","other-visible-kb"]' in injected
|
||||
assert ".breadcrumb-link.dropdown" not in injected
|
||||
assert "reloadForKnowledgeBaseSwitch" in injected
|
||||
assert "function protectedImageApiUrl" in injected
|
||||
assert 'img[data-protected-src]' in injected
|
||||
assert 'encodeURIComponent(cfg.weknoraId)' in injected
|
||||
assert 'credentials: "same-origin"' in injected
|
||||
|
||||
|
||||
def test_weknora_iframe_filters_kb_list_to_signed_visible_mappings() -> None:
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from app.gateway import llmwiki_index_scheduler
|
||||
from deerflow.config.llmwiki_config import LocalWikiEmbeddingConfig, LocalWikiIndexConfig
|
||||
from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient
|
||||
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
|
||||
@@ -51,7 +54,11 @@ class FakeWeKnora:
|
||||
return {"id": knowledge_base_id, "indexing_strategy": {"wiki_enabled": True}}
|
||||
|
||||
async def list_wiki_pages(self, knowledge_base_id: str, *, page: int, page_size: int):
|
||||
return {"pages": [{"slug": item["slug"], "title": item["title"]} for item in self.pages.values()], "total_pages": 1}
|
||||
return {
|
||||
"pages": [{"slug": item["slug"], "title": item["title"]} for item in self.pages.values()],
|
||||
"total": len(self.pages),
|
||||
"total_pages": 1,
|
||||
}
|
||||
|
||||
async def get_wiki_page(self, knowledge_base_id: str, slug: str):
|
||||
return dict(self.pages[slug])
|
||||
@@ -85,6 +92,24 @@ async def test_configured_dimensions_are_validated_but_not_sent_to_bge() -> None
|
||||
assert requests == [{"model": "bge-embedding-m3", "input": ["Wiki page"]}]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_embedding_error_includes_upstream_reason(caplog) -> None:
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(400, json={"detail": "unsupported request parameter: dimensions"})
|
||||
|
||||
config = LocalWikiEmbeddingConfig(
|
||||
model="bge-embedding-m3",
|
||||
base_url="http://embedding.test/v1",
|
||||
max_retries=0,
|
||||
)
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http_client:
|
||||
client = StrictWikiEmbeddingClient(config, http_client=http_client)
|
||||
with pytest.raises(RuntimeError, match="unsupported request parameter: dimensions"):
|
||||
await client.embed_query("Wiki page")
|
||||
|
||||
assert "unsupported request parameter: dimensions" in caplog.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_omitted_dimensions_are_discovered_from_first_response() -> None:
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
@@ -118,6 +143,10 @@ async def test_sync_is_incremental_and_full_scan_deletes_missing_pages() -> None
|
||||
first = await service.sync_mapping(mapping)
|
||||
assert first["changed_pages"] == 1
|
||||
assert embedding.calls == 1
|
||||
statuses = await store.list_page_index_status("kb")
|
||||
assert statuses[0]["title"] == "One"
|
||||
assert statuses[0]["vector_count"] > 0
|
||||
assert store.states["kb"]["remote_page_count"] == 1
|
||||
second = await service.sync_mapping(mapping)
|
||||
assert second["changed_pages"] == 0
|
||||
assert embedding.calls == 1
|
||||
@@ -167,3 +196,39 @@ async def test_fingerprint_change_automatically_rebuilds_entire_mapping() -> Non
|
||||
assert store.states["kb"]["state"] == "idle"
|
||||
assert store.states["kb"]["embedding_fingerprint"] == "fp-new"
|
||||
assert {vector["embedding_fingerprint"] for vectors in store.vectors.values() for vector in vectors} == {"fp-new"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_auto_sync_scheduler_waits_before_first_scan(monkeypatch) -> None:
|
||||
calls: list[str] = []
|
||||
|
||||
async def fake_sleep(seconds: float) -> None:
|
||||
calls.append(f"sleep:{seconds}")
|
||||
raise asyncio.CancelledError
|
||||
|
||||
async def fake_sync_once(app) -> None:
|
||||
calls.append("sync")
|
||||
|
||||
monkeypatch.setattr(llmwiki_index_scheduler.asyncio, "sleep", fake_sleep)
|
||||
monkeypatch.setattr(llmwiki_index_scheduler, "_sync_once", fake_sync_once)
|
||||
app = SimpleNamespace(
|
||||
state=SimpleNamespace(
|
||||
config=SimpleNamespace(
|
||||
llmwiki=SimpleNamespace(
|
||||
local_wiki_index=SimpleNamespace(sync_interval_seconds=120),
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await llmwiki_index_scheduler.local_wiki_index_scheduler_loop(app)
|
||||
|
||||
assert calls == ["sleep:120"]
|
||||
|
||||
|
||||
def test_local_wiki_index_accepts_legacy_sync_flag() -> None:
|
||||
config = LocalWikiIndexConfig.model_validate({"sync_enabled": True})
|
||||
|
||||
assert config.auto_sync is True
|
||||
assert config.sync_enabled is True
|
||||
@@ -0,0 +1,17 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from alembic.config import Config
|
||||
from alembic.script import ScriptDirectory
|
||||
|
||||
|
||||
def test_migration_graph_has_unique_revisions_and_one_head() -> None:
|
||||
migrations = Path(__file__).parents[1] / "packages" / "harness" / "deerflow" / "persistence" / "migrations"
|
||||
config = Config()
|
||||
config.set_main_option("script_location", str(migrations))
|
||||
scripts = ScriptDirectory.from_config(config)
|
||||
|
||||
revisions = list(scripts.walk_revisions())
|
||||
assert len({revision.revision for revision in revisions}) == len(revisions)
|
||||
assert len(scripts.get_heads()) == 1
|
||||
@@ -7,6 +7,7 @@ from fastapi import HTTPException
|
||||
from starlette.requests import Request
|
||||
|
||||
from app.gateway.routers import llmwiki
|
||||
from app.gateway.weknora_embed import verify_weknora_signed_payload
|
||||
|
||||
|
||||
def _request(store: object) -> Request:
|
||||
@@ -177,6 +178,86 @@ async def test_public_wiki_page_list_is_normalized(monkeypatch) -> None:
|
||||
assert payload["pages"][0]["aliases"] == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_public_wiki_file_is_authorized_and_streamed(monkeypatch) -> None:
|
||||
store = PublicStore(_published_row())
|
||||
|
||||
class Client:
|
||||
async def get_knowledge_base_file(self, knowledge_base_id: str, file_path: str):
|
||||
assert knowledge_base_id == "remote-one"
|
||||
assert file_path == "storage://tenant/resource://wiki/figure.png"
|
||||
return b"png-bytes", "image/png"
|
||||
|
||||
monkeypatch.setattr(llmwiki, "_client_or_503", lambda: Client())
|
||||
|
||||
response = await llmwiki.get_public_wiki_file(
|
||||
_request(store),
|
||||
"kb-one",
|
||||
"storage://tenant/resource://wiki/figure.png",
|
||||
)
|
||||
|
||||
assert store.calls == [("kb-one", "__public_wiki__", False, False)]
|
||||
assert response.body == b"png-bytes"
|
||||
assert response.media_type == "image/png"
|
||||
assert response.headers["cache-control"] == "public, max-age=300"
|
||||
assert response.headers["content-disposition"] == "inline; filename*=UTF-8''figure.png"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_private_wiki_file_uses_signed_http_access_instead_of_blob_url(monkeypatch) -> None:
|
||||
async def authorized_mapping(*_args, **_kwargs):
|
||||
return _published_row(), "user-one", False
|
||||
|
||||
class Client:
|
||||
async def get_knowledge_base_file(self, knowledge_base_id: str, file_path: str):
|
||||
assert knowledge_base_id == "remote-one"
|
||||
assert file_path == "storage://tenant/resource://wiki/figure.png"
|
||||
return b"png-bytes", "image/png"
|
||||
|
||||
monkeypatch.setattr(llmwiki, "_authorized_mapping", authorized_mapping)
|
||||
monkeypatch.setattr(llmwiki, "_client_or_503", lambda: Client())
|
||||
|
||||
access = await llmwiki.create_wiki_file_access(
|
||||
_request(PublicStore(_published_row())),
|
||||
"kb-one",
|
||||
"storage://tenant/resource://wiki/figure.png",
|
||||
)
|
||||
payload = verify_weknora_signed_payload(access["token"])
|
||||
assert payload["kind"] == "wiki_file"
|
||||
assert payload["mapping_id"] == "kb-one"
|
||||
assert payload["weknora_id"] == "remote-one"
|
||||
assert payload["file_path"] == "storage://tenant/resource://wiki/figure.png"
|
||||
|
||||
response = await llmwiki.get_signed_wiki_file(access["token"])
|
||||
assert response.body == b"png-bytes"
|
||||
assert response.headers["cache-control"] == "private, max-age=3600"
|
||||
assert response.headers["content-disposition"] == "inline; filename*=UTF-8''figure.png"
|
||||
|
||||
|
||||
def test_wiki_file_proxy_rejects_non_weknora_urls() -> None:
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
llmwiki._validate_protected_wiki_file_path("https://example.test/private.png")
|
||||
|
||||
assert exc_info.value.status_code == 422
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wiki_file_proxy_adds_image_extension_for_opaque_resource_ids(monkeypatch) -> None:
|
||||
class Client:
|
||||
async def get_knowledge_base_file(self, knowledge_base_id: str, file_path: str):
|
||||
return b"png-bytes", "image/png"
|
||||
|
||||
monkeypatch.setattr(llmwiki, "_client_or_503", lambda: Client())
|
||||
|
||||
response = await llmwiki._wiki_file_response(
|
||||
{"weknora_id": "remote-one"},
|
||||
"resource://opaque-image-id",
|
||||
public=True,
|
||||
)
|
||||
|
||||
assert response.headers["content-disposition"] == "inline; filename*=UTF-8''opaque-image-id.png"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_public_wiki_index_preserves_weknora_group_and_item_order(monkeypatch) -> None:
|
||||
store = PublicStore(_published_row())
|
||||
|
||||
Reference in new issue
Block a user