from __future__ import annotations from pathlib import Path import pytest import sqlalchemy as sa from alembic import command from alembic.config import Config from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine from deerflow.persistence.base import Base from deerflow.persistence.llmwiki.memory import MemoryLlmWikiStore from deerflow.persistence.llmwiki.model import LlmWikiKnowledgeBaseRow from deerflow.persistence.llmwiki.sql import LlmWikiRepository _EDGE_INDEXES = { "ix_llmwiki_wiki_edges_knowledge_base_mapping_id", "ix_llmwiki_wiki_edges_source_page_id", "ix_llmwiki_wiki_edges_target_page_id", "ix_llmwiki_wiki_edge_mapping_source", "ix_llmwiki_wiki_edge_mapping_target", } def test_mysql_manual_column_script_keeps_helper_until_all_calls_finish() -> None: sql_path = Path(__file__).parents[1] / "scripts" / "sql" / "column_additions_mysql.sql" sql = sql_path.read_text(encoding="utf-8") drop_offset = sql.index("DROP PROCEDURE IF EXISTS _add_column_if_missing;", sql.index("CREATE PROCEDURE")) call_offsets = [ offset for offset in range(len(sql)) if sql.startswith("CALL _add_column_if_missing(", offset) ] assert call_offsets assert max(call_offsets) < drop_offset assert ( "CALL _add_column_if_missing('llmwiki_wiki_sync_states', " "'last_sync_result', 'TEXT NULL');" ) in sql assert ( "CALL _add_column_if_missing('llmwiki_knowledge_bases', " "'test_questions', 'TEXT NULL');" ) in sql assert ( "CALL _add_column_if_missing('llmwiki_knowledge_bases', " "'retrieval_settings', 'TEXT NULL');" ) in sql assert ( "CALL _add_column_if_missing('llmwiki_knowledge_bases', " "'wiki_parent_directories', 'TEXT NULL');" ) in sql assert ( "CALL _add_column_if_missing('llmwiki_knowledge_bases', " "'wiki_directory_paths', 'TEXT NULL');" ) in sql assert ( "CALL _add_column_if_missing('llmwiki_knowledge_bases', " "'wiki_focus_instructions', 'TEXT NULL');" ) in sql assert ( "CALL _add_column_if_missing('knowledge_retrieval_settings', " "'advanced_management_enabled', 'TINYINT(1) NOT NULL DEFAULT 0');" ) in sql def _config(database: Path) -> Config: migrations = ( Path(__file__).parents[1] / "packages" / "harness" / "deerflow" / "persistence" / "migrations" ) config = Config(str(migrations / "alembic.ini")) config.set_main_option( "sqlalchemy.url", f"sqlite+aiosqlite:///{database.resolve().as_posix()}", ) return config def _prepare_old_database( database: Path, stamp: str, *, has_sync_result: bool, has_retrieval_profile: bool, ) -> None: engine = sa.create_engine(f"sqlite:///{database.resolve().as_posix()}") sync_result = ", last_sync_result JSON" if has_sync_result else "" retrieval_profile = ( ", test_questions JSON, retrieval_settings JSON" if has_retrieval_profile else "" ) with engine.begin() as connection: connection.exec_driver_sql( "CREATE TABLE llmwiki_knowledge_bases " f"(id VARCHAR(36) PRIMARY KEY{retrieval_profile})" ) if has_retrieval_profile: connection.exec_driver_sql( "INSERT INTO llmwiki_knowledge_bases " "(id, test_questions, retrieval_settings) VALUES " "('mapping-1', '[\"保留问题\"]', '{\"top_k\": 8}')" ) connection.exec_driver_sql( "CREATE TABLE llmwiki_wiki_pages (id VARCHAR(36) PRIMARY KEY)" ) connection.exec_driver_sql( "CREATE TABLE llmwiki_wiki_sync_states " f"(id VARCHAR(36) PRIMARY KEY{sync_result})" ) if has_sync_result: connection.exec_driver_sql( "INSERT INTO llmwiki_wiki_sync_states (id, last_sync_result) " "VALUES ('state-1', '{\"processed\": 3}')" ) connection.exec_driver_sql( "CREATE TABLE alembic_version " "(version_num VARCHAR(32) NOT NULL PRIMARY KEY)" ) connection.exec_driver_sql( "INSERT INTO alembic_version (version_num) VALUES (?)", (stamp,), ) engine.dispose() @pytest.mark.parametrize( ("stamp", "has_sync_result", "has_retrieval_profile"), [ ("20260916_01", False, False), ("20260916_02", False, False), ("20260920_01", True, False), ("20260920_01", False, True), ], ) def test_duplicate_historical_stamps_reconcile_to_single_head( tmp_path: Path, stamp: str, has_sync_result: bool, has_retrieval_profile: bool, ) -> None: variant = "sync" if has_sync_result else "profile" if has_retrieval_profile else "base" database = tmp_path / f"old-{stamp}-{variant}.db" _prepare_old_database( database, stamp, has_sync_result=has_sync_result, has_retrieval_profile=has_retrieval_profile, ) config = _config(database) command.upgrade(config, "head") command.upgrade(config, "head") engine = sa.create_engine(f"sqlite:///{database.resolve().as_posix()}") with engine.connect() as connection: inspector = sa.inspect(connection) assert connection.execute( sa.text("SELECT version_num FROM alembic_version") ).scalar_one() == "20261019_01" assert inspector.has_table("llmwiki_wiki_edges") assert _EDGE_INDEXES <= { str(item["name"]) for item in inspector.get_indexes("llmwiki_wiki_edges") } assert "uq_llmwiki_wiki_edge_mapping_pages_type" in { str(item["name"]) for item in inspector.get_unique_constraints("llmwiki_wiki_edges") } assert len(inspector.get_foreign_keys("llmwiki_wiki_edges")) == 3 assert inspector.has_table("model_capabilities") capability_columns = { str(item["name"]): item for item in inspector.get_columns("model_capabilities") } assert { "model_name", "supports_reasoning_effort", "updated_by", "updated_at", } == capability_columns.keys() assert capability_columns["supports_reasoning_effort"]["nullable"] is False mapping_column_info = { str(item["name"]): item for item in inspector.get_columns("llmwiki_knowledge_bases") } assert { "test_questions", "retrieval_settings", "wiki_parent_directories", "wiki_directory_paths", "wiki_focus_instructions", "wiki_content_instructions_raw", "wiki_extraction_instructions_raw", "wiki_generation_managed", "wiki_config_schema_version", "wiki_config_revision", "wiki_config_apply_status", "wiki_config_apply_error", } <= mapping_column_info.keys() assert mapping_column_info["wiki_parent_directories"]["nullable"] is True assert mapping_column_info["wiki_directory_paths"]["nullable"] is True retrieval_setting_columns = { str(item["name"]): item for item in inspector.get_columns("knowledge_retrieval_settings") } assert "advanced_management_enabled" in retrieval_setting_columns assert retrieval_setting_columns["advanced_management_enabled"]["nullable"] is False if has_retrieval_profile: assert connection.execute( sa.text( "SELECT test_questions FROM llmwiki_knowledge_bases " "WHERE id = 'mapping-1'" ) ).scalar_one() == '["保留问题"]' assert connection.execute( sa.text( "SELECT retrieval_settings FROM llmwiki_knowledge_bases " "WHERE id = 'mapping-1'" ) ).scalar_one() == '{"top_k": 8}' sync_columns = { str(item["name"]) for item in inspector.get_columns("llmwiki_wiki_sync_states") } assert "last_sync_result" in sync_columns if has_sync_result: assert connection.execute( sa.text( "SELECT last_sync_result FROM llmwiki_wiki_sync_states " "WHERE id = 'state-1'" ) ).scalar_one() == '{"processed": 3}' engine.dispose() async def _exercise_parent_directories(store) -> str: configured = ["专题资料", "政策法规"] directory_paths = [["专题资料"], ["专题资料", "国内政策"]] retrieval_settings = { "entity_expansion_enabled": True, "max_related_terms": 6, "statistics_enabled": True, } created = await store.create_mapping( weknora_id="remote-parent-directories", owner_user_id="user-one", name="知识库", description="", kb_type="document", retrieval_settings=retrieval_settings, wiki_parent_directories=configured, wiki_directory_paths=directory_paths, wiki_focus_instructions="重点提取适用对象", wiki_content_instructions_raw="保留关键数字", wiki_extraction_instructions_raw="抽取时间线", wiki_generation_managed=True, wiki_config_schema_version=2, wiki_config_revision=1, wiki_config_apply_status="active", ) mapping_id = str(created["id"]) assert created["wiki_parent_directories"] == configured assert all(isinstance(item, str) for item in created["wiki_parent_directories"]) assert created["wiki_directory_paths"] == directory_paths assert created["wiki_focus_instructions"] == "重点提取适用对象" assert created["wiki_generation_managed"] is True assert created["wiki_config_revision"] == 1 assert created["retrieval_settings"] == retrieval_settings configured.append("调用方后续修改") directory_paths[1].append("调用方后续修改") retrieval_settings["max_related_terms"] = 12 unchanged = await store.update_mapping(mapping_id, description="已更新") assert unchanged is not None assert unchanged["wiki_parent_directories"] == ["专题资料", "政策法规"] assert unchanged["wiki_directory_paths"] == [ ["专题资料"], ["专题资料", "国内政策"], ] assert unchanged["retrieval_settings"]["max_related_terms"] == 6 emptied = await store.update_mapping(mapping_id, wiki_parent_directories=[]) assert emptied is not None assert emptied["wiki_parent_directories"] == [] reset = await store.update_mapping( mapping_id, wiki_parent_directories=["重新设置"], wiki_directory_paths=[["重新设置"], ["重新设置", "二级"]], wiki_focus_instructions="新重点", wiki_config_revision=2, ) assert reset is not None assert reset["wiki_parent_directories"] == ["重新设置"] assert reset["wiki_directory_paths"] == [["重新设置"], ["重新设置", "二级"]] assert reset["wiki_focus_instructions"] == "新重点" assert reset["wiki_config_revision"] == 2 cleared = await store.update_mapping(mapping_id, wiki_parent_directories=None) assert cleared is not None assert cleared["wiki_parent_directories"] is None return mapping_id @pytest.mark.asyncio async def test_memory_llmwiki_parent_directories_persistence() -> None: store = MemoryLlmWikiStore() mapping_id = await _exercise_parent_directories(store) saved = await store.get_authorized( mapping_id, "user-one", write=True, is_admin=False, ) assert saved is not None assert saved["wiki_parent_directories"] is None @pytest.mark.asyncio async def test_sql_llmwiki_parent_directories_persistence(tmp_path: Path) -> None: engine = create_async_engine( f"sqlite+aiosqlite:///{tmp_path / 'llmwiki-parent-directories.db'}" ) async with engine.begin() as connection: await connection.run_sync( lambda sync_connection: Base.metadata.create_all( sync_connection, tables=[LlmWikiKnowledgeBaseRow.__table__], ) ) session_factory = async_sessionmaker(engine, expire_on_commit=False) store = LlmWikiRepository(session_factory) mapping_id = await _exercise_parent_directories(store) async with session_factory() as session: persisted = await session.scalar( sa.select(LlmWikiKnowledgeBaseRow.wiki_parent_directories).where( LlmWikiKnowledgeBaseRow.id == mapping_id ) ) assert persisted is None await engine.dispose()