fix: harden wiki delivery and leaderboard performance

This commit is contained in:
吉鑫升 committed 2026-08-25 16:57:42 +08:00
1 parent 0c5e2e8cb0
commit 9f29ecae9e
28 files changed
+616 -68

No files matched your search

@@ -0,0 +1,43 @@
from __future__ import annotations
from sqlalchemy import select
from sqlalchemy.dialects import mysql
from app.gateway.routers.admin_users import _apply_run_scope, _apply_tool_scope
from deerflow.persistence.run.model import RunRow
from deerflow.persistence.tool_metrics.model import ToolCallMetricRow
def _mysql_sql(stmt) -> str:
return str(stmt.compile(dialect=mysql.dialect(), compile_kwargs={"literal_binds": True})).upper()
def test_excluding_scheduled_runs_uses_index_friendly_not_exists() -> None:
stmt = _apply_run_scope(
select(RunRow.run_id),
user_column=RunRow.user_id,
run_id_column=RunRow.run_id,
exclude_user_ids=set(),
include_admins=True,
include_failed=True,
include_scheduled=False,
)
sql = _mysql_sql(stmt)
assert "NOT (EXISTS" in sql
assert "SCHEDULED_TASK_RUNS" in sql
assert " NOT IN " not in sql
def test_tool_aggregation_does_not_join_the_full_runs_table() -> None:
stmt = _apply_tool_scope(
select(ToolCallMetricRow.tool_name),
exclude_user_ids=set(),
include_admins=True,
include_scheduled=False,
)
sql = _mysql_sql(stmt)
assert sql.count("NOT (EXISTS") == 2
assert "LEFT OUTER JOIN" not in sql
assert " NOT IN " not in sql
@@ -66,6 +66,22 @@ async def test_sql_store_atomically_replaces_vectors_and_marks_missing_deleted(t
assert store.write_batch_size == 1
assert len(snapshot["rows"]) == 3
assert decode_vector(snapshot["rows"][0]["vector_blob"], 2).tolist() == pytest.approx([0.6, 0.8])
page_statuses = await store.list_page_index_status("mapping-1")
assert len(page_statuses) == 1
assert page_statuses[0] == {
**page_statuses[0],
"id": saved["id"],
"slug": "people/alice",
"title": "Alice",
"page_type": "article",
"status": "published",
"index_status": "ready",
"vector_count": 3,
"last_error": None,
"last_seen_sync_id": "sync-1",
}
assert page_statuses[0]["indexed_at"]
assert page_statuses[0]["updated_at"]
assert await store.mark_missing_pages_deleted("mapping-1", "sync-2") == 1
assert (await store.load_vector_snapshot("mapping-1", "fp"))["rows"] == []
@@ -54,7 +54,11 @@ class FakeWeKnora:
return {"id": knowledge_base_id, "indexing_strategy": {"wiki_enabled": True}}
async def list_wiki_pages(self, knowledge_base_id: str, *, page: int, page_size: int):
return {"pages": [{"slug": item["slug"], "title": item["title"]} for item in self.pages.values()], "total_pages": 1}
return {
"pages": [{"slug": item["slug"], "title": item["title"]} for item in self.pages.values()],
"total": len(self.pages),
"total_pages": 1,
}
async def get_wiki_page(self, knowledge_base_id: str, slug: str):
return dict(self.pages[slug])
@@ -139,6 +143,10 @@ async def test_sync_is_incremental_and_full_scan_deletes_missing_pages() -> None
first = await service.sync_mapping(mapping)
assert first["changed_pages"] == 1
assert embedding.calls == 1
statuses = await store.list_page_index_status("kb")
assert statuses[0]["title"] == "One"
assert statuses[0]["vector_count"] > 0
assert store.states["kb"]["remote_page_count"] == 1
second = await service.sync_mapping(mapping)
assert second["changed_pages"] == 0
assert embedding.calls == 1
@@ -0,0 +1,17 @@
from __future__ import annotations
from pathlib import Path
from alembic.config import Config
from alembic.script import ScriptDirectory
def test_migration_graph_has_unique_revisions_and_one_head() -> None:
migrations = Path(__file__).parents[1] / "packages" / "harness" / "deerflow" / "persistence" / "migrations"
config = Config()
config.set_main_option("script_location", str(migrations))
scripts = ScriptDirectory.from_config(config)
revisions = list(scripts.walk_revisions())
assert len({revision.revision for revision in revisions}) == len(revisions)
assert len(scripts.get_heads()) == 1
@@ -7,6 +7,7 @@ from fastapi import HTTPException
from starlette.requests import Request
from app.gateway.routers import llmwiki
from app.gateway.weknora_embed import verify_weknora_signed_payload
def _request(store: object) -> Request:
@@ -199,6 +200,38 @@ async def test_public_wiki_file_is_authorized_and_streamed(monkeypatch) -> None:
assert response.body == b"png-bytes"
assert response.media_type == "image/png"
assert response.headers["cache-control"] == "public, max-age=300"
assert response.headers["content-disposition"] == "inline; filename*=UTF-8''figure.png"
@pytest.mark.asyncio
async def test_private_wiki_file_uses_signed_http_access_instead_of_blob_url(monkeypatch) -> None:
async def authorized_mapping(*_args, **_kwargs):
return _published_row(), "user-one", False
class Client:
async def get_knowledge_base_file(self, knowledge_base_id: str, file_path: str):
assert knowledge_base_id == "remote-one"
assert file_path == "storage://tenant/resource://wiki/figure.png"
return b"png-bytes", "image/png"
monkeypatch.setattr(llmwiki, "_authorized_mapping", authorized_mapping)
monkeypatch.setattr(llmwiki, "_client_or_503", lambda: Client())
access = await llmwiki.create_wiki_file_access(
_request(PublicStore(_published_row())),
"kb-one",
"storage://tenant/resource://wiki/figure.png",
)
payload = verify_weknora_signed_payload(access["token"])
assert payload["kind"] == "wiki_file"
assert payload["mapping_id"] == "kb-one"
assert payload["weknora_id"] == "remote-one"
assert payload["file_path"] == "storage://tenant/resource://wiki/figure.png"
response = await llmwiki.get_signed_wiki_file(access["token"])
assert response.body == b"png-bytes"
assert response.headers["cache-control"] == "private, max-age=3600"
assert response.headers["content-disposition"] == "inline; filename*=UTF-8''figure.png"
def test_wiki_file_proxy_rejects_non_weknora_urls() -> None:
@@ -208,6 +241,23 @@ def test_wiki_file_proxy_rejects_non_weknora_urls() -> None:
assert exc_info.value.status_code == 422
@pytest.mark.asyncio
async def test_wiki_file_proxy_adds_image_extension_for_opaque_resource_ids(monkeypatch) -> None:
class Client:
async def get_knowledge_base_file(self, knowledge_base_id: str, file_path: str):
return b"png-bytes", "image/png"
monkeypatch.setattr(llmwiki, "_client_or_503", lambda: Client())
response = await llmwiki._wiki_file_response(
{"weknora_id": "remote-one"},
"resource://opaque-image-id",
public=True,
)
assert response.headers["content-disposition"] == "inline; filename*=UTF-8''opaque-image-id.png"
@pytest.mark.asyncio
async def test_public_wiki_index_preserves_weknora_group_and_item_order(monkeypatch) -> None:
store = PublicStore(_published_row())