feat: improve assistant knowledge import and database compatibility
This commit is contained in:
1 parent
fbd8dec48f
commit
6b724bd00d
51 files changed
+4393
-1021
No files matched your search
@@ -22,6 +22,18 @@ class _FakeThreadStore:
|
||||
return self.rows[offset : offset + limit]
|
||||
|
||||
|
||||
class _FakeAgentStore:
|
||||
async def list_by_ids(self, agent_ids: list[str], user_id: str) -> list[dict]:
|
||||
assert user_id == "owner"
|
||||
agents = {
|
||||
"analysis-agent": {
|
||||
"id": "analysis-agent",
|
||||
"name": "舆情分析智能体",
|
||||
}
|
||||
}
|
||||
return [agents[agent_id] for agent_id in agent_ids if agent_id in agents]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_artifact_library_aggregates_generated_non_media_files(tmp_path: Path, monkeypatch):
|
||||
thread_one = tmp_path / "thread-one" / "outputs"
|
||||
@@ -79,6 +91,7 @@ async def test_artifact_library_aggregates_generated_non_media_files(tmp_path: P
|
||||
|
||||
assert [item["name"] for item in items] == ["证据.zip", "结论.md", "数据汇总.xlsx", "研判报告.docx"]
|
||||
assert [item["file_kind"] for item in items] == ["archive", "text_data", "spreadsheet", "document"]
|
||||
assert [item["source_type"] for item in items] == ["general_qa", "general_qa", "agent", "agent"]
|
||||
assert items[-1]["thread_title"] == "战场环境综合研判"
|
||||
assert items[-1]["agent_id"] == "analysis-agent"
|
||||
assert all(item["name"] not in {"配图.png", "录音.mp3"} for item in items)
|
||||
@@ -88,9 +101,12 @@ async def test_artifact_library_aggregates_generated_non_media_files(tmp_path: P
|
||||
@pytest.mark.asyncio
|
||||
async def test_artifact_library_endpoint_filters_searches_and_pages(tmp_path: Path, monkeypatch):
|
||||
output_dir = tmp_path / "thread-one" / "outputs"
|
||||
agent_output_dir = tmp_path / "thread-agent" / "outputs"
|
||||
output_dir.mkdir(parents=True)
|
||||
agent_output_dir.mkdir(parents=True)
|
||||
(output_dir / "任务报告.md").write_text("report", encoding="utf-8")
|
||||
(output_dir / "任务数据.csv").write_text("a,b", encoding="utf-8")
|
||||
(agent_output_dir / "智能体结论.md").write_text("agent report", encoding="utf-8")
|
||||
|
||||
store = _FakeThreadStore(
|
||||
[
|
||||
@@ -98,7 +114,12 @@ async def test_artifact_library_endpoint_filters_searches_and_pages(tmp_path: Pa
|
||||
"thread_id": "thread-one",
|
||||
"display_name": "任务研判对话",
|
||||
"metadata": {},
|
||||
}
|
||||
},
|
||||
{
|
||||
"thread_id": "thread-agent",
|
||||
"display_name": "智能体研判对话",
|
||||
"metadata": {"agent_id": "analysis-agent"},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
@@ -113,6 +134,7 @@ async def test_artifact_library_endpoint_filters_searches_and_pages(tmp_path: Pa
|
||||
monkeypatch.setattr(artifacts, "aresolve_path_user_id", _resolve_owner)
|
||||
monkeypatch.setattr(artifacts, "get_effective_user_id", lambda: "owner")
|
||||
monkeypatch.setattr(deps, "get_thread_store", lambda _request: store)
|
||||
monkeypatch.setattr(deps, "get_agent_store", lambda _request: _FakeAgentStore())
|
||||
artifacts._artifact_library_cache.clear()
|
||||
|
||||
raw_endpoint = artifacts.list_artifact_library.__wrapped__
|
||||
@@ -120,6 +142,7 @@ async def test_artifact_library_endpoint_filters_searches_and_pages(tmp_path: Pa
|
||||
request=SimpleNamespace(),
|
||||
query="研判对话",
|
||||
file_kind="text_data",
|
||||
source_type="general_qa",
|
||||
offset=0,
|
||||
limit=1,
|
||||
refresh=True,
|
||||
@@ -134,9 +157,24 @@ async def test_artifact_library_endpoint_filters_searches_and_pages(tmp_path: Pa
|
||||
request=SimpleNamespace(),
|
||||
query="任务数据",
|
||||
file_kind="spreadsheet",
|
||||
source_type="all",
|
||||
offset=0,
|
||||
limit=50,
|
||||
refresh=False,
|
||||
)
|
||||
assert spreadsheet_response.total == 1
|
||||
assert spreadsheet_response.items[0].name == "任务数据.csv"
|
||||
|
||||
agent_response = await raw_endpoint(
|
||||
request=SimpleNamespace(),
|
||||
query=None,
|
||||
file_kind="all",
|
||||
source_type="agent",
|
||||
offset=0,
|
||||
limit=50,
|
||||
refresh=False,
|
||||
)
|
||||
assert agent_response.total == 1
|
||||
assert agent_response.items[0].source_type == "agent"
|
||||
assert agent_response.items[0].agent_name == "舆情分析智能体"
|
||||
assert agent_response.items[0].name == "智能体结论.md"
|
||||
@@ -11,6 +11,13 @@ from deerflow.persistence.assistant_knowledge.model import (
|
||||
from deerflow.persistence.base import Base
|
||||
|
||||
|
||||
class FakeEmbeddingClient:
|
||||
dimensions = 3
|
||||
|
||||
async def embed_query(self, _query: str) -> list[float]:
|
||||
return [1.0, 0.0, 0.0]
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def repository():
|
||||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||||
@@ -94,3 +101,93 @@ async def test_delete_source_keeps_entity_until_last_contribution_is_removed(
|
||||
async with repository._sf() as session:
|
||||
entity = (await session.execute(select(AssistantKnowledgeEntityRow))).scalar_one()
|
||||
assert entity.status == "deleted"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_custom_base_import_is_also_mirrored_to_global_base(
|
||||
repository: AssistantKnowledgeRepository,
|
||||
) -> None:
|
||||
global_base = await repository.initialize(created_by="admin")
|
||||
custom_base = await repository.create_base(
|
||||
created_by="admin",
|
||||
name="专题助手库",
|
||||
description="只承接某个专题的 WeKnora 导出数据",
|
||||
)
|
||||
package = {
|
||||
"source": {"type": "weknora"},
|
||||
"wiki_pages": [{"slug": "wiki/topic", "title": "专题 Wiki", "content": "王五负责统筹。"}],
|
||||
"entities": [{"local_id": "e1", "type": "person", "name": "王五"}],
|
||||
"relations": [],
|
||||
}
|
||||
|
||||
job = await repository.import_package(
|
||||
package=package,
|
||||
source_type="weknora",
|
||||
source_key="kb-topic",
|
||||
source_name="专题普通库",
|
||||
trigger="manual_from_weknora",
|
||||
created_by="admin",
|
||||
mapping_id="kb-topic",
|
||||
base_id=custom_base["id"],
|
||||
)
|
||||
|
||||
assert job["base_id"] == custom_base["id"]
|
||||
assert job["mirrored_global_base_id"] == global_base["id"]
|
||||
assert len(await repository.list_pages(base_id=custom_base["id"], query="王五")) == 2
|
||||
assert len(await repository.list_pages(base_id=global_base["id"], query="王五")) == 2
|
||||
bases = await repository.list_bases()
|
||||
assert [base["id"] for base in bases[:2]] == [global_base["id"], custom_base["id"]]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_imported_vectors_are_reused_for_assistant_search(
|
||||
repository: AssistantKnowledgeRepository,
|
||||
) -> None:
|
||||
base = await repository.initialize(created_by="admin")
|
||||
job = await repository.import_package(
|
||||
package={
|
||||
"source": {"type": "weknora"},
|
||||
"wiki_pages": [{"slug": "wiki/vector", "title": "向量页", "content": "页面正文"}],
|
||||
"vectors": [
|
||||
{
|
||||
"wiki_slug": "wiki/vector",
|
||||
"section_index": 0,
|
||||
"section_content": "张三负责向量化检索。",
|
||||
"embedding_model": "bge-embedding-m3",
|
||||
"embedding_dimensions": 3,
|
||||
"vector": [1.0, 0.0, 0.0],
|
||||
},
|
||||
{
|
||||
"wiki_slug": "wiki/vector",
|
||||
"section_index": 1,
|
||||
"section_content": "无关片段。",
|
||||
"embedding_model": "bge-embedding-m3",
|
||||
"embedding_dimensions": 3,
|
||||
"vector": [0.0, 1.0, 0.0],
|
||||
},
|
||||
],
|
||||
},
|
||||
source_type="weknora",
|
||||
source_key="kb-vector",
|
||||
source_name="向量普通库",
|
||||
trigger="manual_package_upload",
|
||||
created_by="admin",
|
||||
)
|
||||
|
||||
assert job["counts"]["vectors"] == 2
|
||||
results = await repository.search_pages(
|
||||
base_ids=[base["id"]],
|
||||
query="这个查询不靠关键词",
|
||||
embedding_client=FakeEmbeddingClient(),
|
||||
)
|
||||
assert results[0]["retrieval_mode"] == "vector"
|
||||
assert results[0]["matched_section"]["embedding_model"] == "bge-embedding-m3"
|
||||
assert results[0]["content_markdown"] == "张三负责向量化检索。"
|
||||
|
||||
sources = await repository.list_sources()
|
||||
assert await repository.delete_source(sources[0]["id"])
|
||||
assert await repository.search_pages(
|
||||
base_ids=[base["id"]],
|
||||
query="这个查询不靠关键词",
|
||||
embedding_client=FakeEmbeddingClient(),
|
||||
) == []
|
||||
@@ -0,0 +1,81 @@
|
||||
"""Regression coverage for the openGauss/GaussDB persistence backend."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
from sqlalchemy.engine import make_url
|
||||
from sqlalchemy.schema import CreateIndex, CreateTable
|
||||
|
||||
from deerflow.config.database_config import DatabaseConfig
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("configured", "expected"),
|
||||
[
|
||||
(
|
||||
"opengauss://user:pass@127.0.0.1:26000/deerflow",
|
||||
"opengauss+asyncpg://user:pass@127.0.0.1:26000/deerflow",
|
||||
),
|
||||
(
|
||||
"postgresql://user:pass@127.0.0.1:26000/deerflow?ssl=require",
|
||||
"opengauss+asyncpg://user:pass@127.0.0.1:26000/deerflow?ssl=require",
|
||||
),
|
||||
(
|
||||
"opengauss+asyncpg://user:pass@127.0.0.1:26000/deerflow",
|
||||
"opengauss+asyncpg://user:pass@127.0.0.1:26000/deerflow",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_gauss_urls_always_select_the_opengauss_async_dialect(
|
||||
configured: str,
|
||||
expected: str,
|
||||
) -> None:
|
||||
config = DatabaseConfig(backend="gauss", gauss_url=configured)
|
||||
assert config.app_sqlalchemy_url == expected
|
||||
|
||||
|
||||
def test_gauss_backend_requires_a_supported_nonempty_url() -> None:
|
||||
with pytest.raises(ValueError, match="gauss_url is required"):
|
||||
_ = DatabaseConfig(backend="gauss").app_sqlalchemy_url
|
||||
|
||||
with pytest.raises(ValueError, match="must use opengauss"):
|
||||
_ = DatabaseConfig(
|
||||
backend="gauss",
|
||||
gauss_url="mysql://user:pass@127.0.0.1/deerflow",
|
||||
).app_sqlalchemy_url
|
||||
|
||||
with pytest.raises(ValidationError):
|
||||
DatabaseConfig(backend="unknown")
|
||||
|
||||
|
||||
def test_gauss_timestamps_keep_timezone_information() -> None:
|
||||
dialect = type("GaussDialect", (), {"name": "opengauss"})()
|
||||
value = datetime(2026, 9, 2, 8, 30, tzinfo=UTC)
|
||||
column_type = BeijingDateTime()
|
||||
|
||||
assert column_type.process_bind_param(value, dialect) is value
|
||||
assert column_type.process_result_value(value, dialect) is value
|
||||
|
||||
|
||||
def test_official_async_dialect_compiles_the_complete_orm_schema() -> None:
|
||||
"""Every registered table/index must compile before startup touches GaussDB."""
|
||||
pytest.importorskip("opengauss_sqlalchemy")
|
||||
|
||||
import deerflow.persistence.models # noqa: F401 -- register ORM metadata
|
||||
from deerflow.persistence.base import Base
|
||||
|
||||
dialect_cls = make_url("opengauss+asyncpg://user:pass@localhost/db").get_dialect(
|
||||
_is_async=True
|
||||
)
|
||||
dialect = dialect_cls()
|
||||
|
||||
assert dialect.__class__.__module__.startswith("opengauss_sqlalchemy")
|
||||
assert Base.metadata.sorted_tables
|
||||
for table in Base.metadata.sorted_tables:
|
||||
assert str(CreateTable(table).compile(dialect=dialect)).startswith("\nCREATE TABLE")
|
||||
for index in table.indexes:
|
||||
assert str(CreateIndex(index).compile(dialect=dialect)).startswith("CREATE")
|
||||
@@ -66,6 +66,9 @@ async def test_sql_store_atomically_replaces_vectors_and_marks_missing_deleted(t
|
||||
assert store.write_batch_size == 1
|
||||
assert len(snapshot["rows"]) == 3
|
||||
assert decode_vector(snapshot["rows"][0]["vector_blob"], 2).tolist() == pytest.approx([0.6, 0.8])
|
||||
streamed = [row async for row in store.iter_vector_snapshot_rows("mapping-1", "fp", batch_size=2)]
|
||||
assert [row["section_index"] for row in streamed] == [0, 1, 2]
|
||||
assert streamed[0]["page"]["slug"] == "people/alice"
|
||||
page_statuses = await store.list_page_index_status("mapping-1")
|
||||
assert len(page_statuses) == 1
|
||||
assert page_statuses[0] == {
|
||||
|
||||
Reference in new issue
Block a user