sync: 鍚屾灞€鍩熺綉鏈€鏂颁唬鐮?(source master @ b69568e4)

This commit is contained in:
gozeros committed 2026-10-21 14:51:31 +08:00
1 parent 4f7f07504a
commit 8bc8b0e082
1651 files changed
+278599 -11416

No files matched your search

@@ -6,11 +6,47 @@ from abc import ABC, abstractmethod
from typing import Any, Literal
LlmWikiScope = Literal["personal", "public", "all"]
UNSET = object()
def serialize_wiki_parent_directories(
value: list[str] | None,
) -> list[str] | None:
return None if value is None else [str(item) for item in value]
def serialize_wiki_directory_paths(
value: list[list[str]] | None,
) -> list[list[str]] | None:
return (
None
if value is None
else [[str(segment) for segment in path] for path in value]
)
class LlmWikiStore(ABC):
@abstractmethod
async def create_mapping(self, *, weknora_id: str, owner_user_id: str, name: str, description: str, kb_type: str) -> dict[str, Any]: ...
async def create_mapping(
self,
*,
weknora_id: str,
owner_user_id: str,
name: str,
description: str,
kb_type: str,
retrieval_settings: dict[str, Any] | None = None,
wiki_parent_directories: list[str] | None = None,
wiki_directory_paths: list[list[str]] | None = None,
wiki_focus_instructions: str | None = None,
wiki_content_instructions_raw: str | None = None,
wiki_extraction_instructions_raw: str | None = None,
wiki_generation_managed: bool | None = None,
wiki_config_schema_version: int | None = None,
wiki_config_revision: int | None = None,
wiki_config_apply_status: str | None = None,
wiki_config_apply_error: str | None = None,
) -> dict[str, Any]: ...
@abstractmethod
async def list_visible(self, user_id: str, *, scope: LlmWikiScope = "all", is_admin: bool = False) -> list[dict[str, Any]]: ...
@@ -28,8 +64,21 @@ class LlmWikiStore(ABC):
*,
name: str | None = None,
description: str | None = None,
test_questions: list[str] | None = None,
test_questions_set: bool = False,
retrieval_settings: dict[str, Any] | None = None,
wiki_index_enabled: bool | None = None,
external_search_enabled: bool | None = None,
wiki_parent_directories: list[str] | None | object = UNSET,
wiki_directory_paths: list[list[str]] | None | object = UNSET,
wiki_focus_instructions: str | None | object = UNSET,
wiki_content_instructions_raw: str | None | object = UNSET,
wiki_extraction_instructions_raw: str | None | object = UNSET,
wiki_generation_managed: bool | None | object = UNSET,
wiki_config_schema_version: int | None | object = UNSET,
wiki_config_revision: int | None | object = UNSET,
wiki_config_apply_status: str | None | object = UNSET,
wiki_config_apply_error: str | None | object = UNSET,
) -> dict[str, Any] | None: ...
@abstractmethod
@@ -3,22 +3,58 @@
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from typing import Any, cast
from uuid import uuid4
from deerflow.persistence.llmwiki.base import LlmWikiScope, LlmWikiStore
from deerflow.persistence.llmwiki.base import (
UNSET,
LlmWikiScope,
LlmWikiStore,
serialize_wiki_directory_paths,
serialize_wiki_parent_directories,
)
def _now() -> str:
return datetime.now(UTC).isoformat()
def _mapping_to_dict(row: dict[str, Any]) -> dict[str, Any]:
data = dict(row)
data["wiki_parent_directories"] = serialize_wiki_parent_directories(
data.get("wiki_parent_directories")
)
data["wiki_directory_paths"] = serialize_wiki_directory_paths(
data.get("wiki_directory_paths")
)
return data
class MemoryLlmWikiStore(LlmWikiStore):
def __init__(self) -> None:
self._rows: dict[str, dict[str, Any]] = {}
self._conversation_deposits: dict[str, dict[str, Any]] = {}
async def create_mapping(self, *, weknora_id: str, owner_user_id: str, name: str, description: str, kb_type: str) -> dict[str, Any]:
async def create_mapping(
self,
*,
weknora_id: str,
owner_user_id: str,
name: str,
description: str,
kb_type: str,
retrieval_settings: dict[str, Any] | None = None,
wiki_parent_directories: list[str] | None = None,
wiki_directory_paths: list[list[str]] | None = None,
wiki_focus_instructions: str | None = None,
wiki_content_instructions_raw: str | None = None,
wiki_extraction_instructions_raw: str | None = None,
wiki_generation_managed: bool | None = None,
wiki_config_schema_version: int | None = None,
wiki_config_revision: int | None = None,
wiki_config_apply_status: str | None = None,
wiki_config_apply_error: str | None = None,
) -> dict[str, Any]:
if any(row["weknora_id"] == weknora_id for row in self._rows.values()):
raise ValueError("WeKnora knowledge base is already mapped")
now = _now()
@@ -28,6 +64,26 @@ class MemoryLlmWikiStore(LlmWikiStore):
"owner_user_id": owner_user_id,
"name": name,
"description": description,
"test_questions": None,
"retrieval_settings": (
dict(retrieval_settings)
if retrieval_settings is not None
else None
),
"wiki_parent_directories": serialize_wiki_parent_directories(
wiki_parent_directories
),
"wiki_directory_paths": serialize_wiki_directory_paths(
wiki_directory_paths
),
"wiki_focus_instructions": wiki_focus_instructions,
"wiki_content_instructions_raw": wiki_content_instructions_raw,
"wiki_extraction_instructions_raw": wiki_extraction_instructions_raw,
"wiki_generation_managed": wiki_generation_managed,
"wiki_config_schema_version": wiki_config_schema_version,
"wiki_config_revision": wiki_config_revision,
"wiki_config_apply_status": wiki_config_apply_status,
"wiki_config_apply_error": wiki_config_apply_error,
"kb_type": kb_type,
"publication_status": "private",
"published_by": None,
@@ -40,7 +96,7 @@ class MemoryLlmWikiStore(LlmWikiStore):
"updated_at": now,
}
self._rows[row["id"]] = row
return dict(row)
return _mapping_to_dict(row)
async def list_visible(self, user_id: str, *, scope: LlmWikiScope = "all", is_admin: bool = False) -> list[dict[str, Any]]:
rows = list(self._rows.values())
@@ -50,22 +106,22 @@ class MemoryLlmWikiStore(LlmWikiStore):
rows = [row for row in rows if row["publication_status"] == "published"]
else:
rows = rows if is_admin else [row for row in rows if row["owner_user_id"] == user_id or row["publication_status"] == "published"]
return [dict(row) for row in sorted(rows, key=lambda row: row["created_at"], reverse=True)]
return [_mapping_to_dict(row) for row in sorted(rows, key=lambda row: row["created_at"], reverse=True)]
async def get_authorized(self, mapping_id: str, user_id: str, *, write: bool, is_admin: bool) -> dict[str, Any] | None:
row = self._rows.get(mapping_id)
if row is None:
return None
if is_admin or row["owner_user_id"] == user_id:
return dict(row)
return _mapping_to_dict(row)
if not write and row["publication_status"] == "published":
return dict(row)
return _mapping_to_dict(row)
return None
async def get_by_weknora_id(self, weknora_id: str) -> dict[str, Any] | None:
for row in self._rows.values():
if row["weknora_id"] == weknora_id:
return dict(row)
return _mapping_to_dict(row)
return None
async def update_mapping(
@@ -74,8 +130,21 @@ class MemoryLlmWikiStore(LlmWikiStore):
*,
name: str | None = None,
description: str | None = None,
test_questions: list[str] | None = None,
test_questions_set: bool = False,
retrieval_settings: dict[str, Any] | None = None,
wiki_index_enabled: bool | None = None,
external_search_enabled: bool | None = None,
wiki_parent_directories: list[str] | None | object = UNSET,
wiki_directory_paths: list[list[str]] | None | object = UNSET,
wiki_focus_instructions: str | None | object = UNSET,
wiki_content_instructions_raw: str | None | object = UNSET,
wiki_extraction_instructions_raw: str | None | object = UNSET,
wiki_generation_managed: bool | None | object = UNSET,
wiki_config_schema_version: int | None | object = UNSET,
wiki_config_revision: int | None | object = UNSET,
wiki_config_apply_status: str | None | object = UNSET,
wiki_config_apply_error: str | None | object = UNSET,
) -> dict[str, Any] | None:
row = self._rows.get(mapping_id)
if row is None:
@@ -84,13 +153,65 @@ class MemoryLlmWikiStore(LlmWikiStore):
row["name"] = name
if description is not None:
row["description"] = description
if test_questions_set:
row["test_questions"] = list(test_questions or [])
if retrieval_settings is not None:
row["retrieval_settings"] = dict(retrieval_settings)
if wiki_index_enabled is not None:
row["wiki_index_enabled"] = wiki_index_enabled
if external_search_enabled is not None:
row["external_search_enabled"] = external_search_enabled
row["external_search_updated_at"] = _now()
if wiki_parent_directories is not UNSET:
row["wiki_parent_directories"] = serialize_wiki_parent_directories(
cast(list[str] | None, wiki_parent_directories)
)
if wiki_directory_paths is not UNSET:
row["wiki_directory_paths"] = serialize_wiki_directory_paths(
cast(list[list[str]] | None, wiki_directory_paths)
)
if wiki_focus_instructions is not UNSET:
row["wiki_focus_instructions"] = cast(
str | None,
wiki_focus_instructions,
)
if wiki_content_instructions_raw is not UNSET:
row["wiki_content_instructions_raw"] = cast(
str | None,
wiki_content_instructions_raw,
)
if wiki_extraction_instructions_raw is not UNSET:
row["wiki_extraction_instructions_raw"] = cast(
str | None,
wiki_extraction_instructions_raw,
)
if wiki_generation_managed is not UNSET:
row["wiki_generation_managed"] = cast(
bool | None,
wiki_generation_managed,
)
if wiki_config_schema_version is not UNSET:
row["wiki_config_schema_version"] = cast(
int | None,
wiki_config_schema_version,
)
if wiki_config_revision is not UNSET:
row["wiki_config_revision"] = cast(
int | None,
wiki_config_revision,
)
if wiki_config_apply_status is not UNSET:
row["wiki_config_apply_status"] = cast(
str | None,
wiki_config_apply_status,
)
if wiki_config_apply_error is not UNSET:
row["wiki_config_apply_error"] = cast(
str | None,
wiki_config_apply_error,
)
row["updated_at"] = _now()
return dict(row)
return _mapping_to_dict(row)
async def adopt_conversation_deposit_mapping(
self,
@@ -115,7 +236,7 @@ class MemoryLlmWikiStore(LlmWikiStore):
"updated_at": now,
}
)
return dict(row)
return _mapping_to_dict(row)
async def delete_mapping(self, mapping_id: str) -> bool:
return self._rows.pop(mapping_id, None) is not None
@@ -129,7 +250,7 @@ class MemoryLlmWikiStore(LlmWikiStore):
row["reviewed_by"] = None
row["published_at"] = _now()
row["updated_at"] = _now()
return dict(row)
return _mapping_to_dict(row)
async def review_publish(self, mapping_id: str, reviewer_user_id: str, *, approved: bool) -> dict[str, Any] | None:
row = self._rows.get(mapping_id)
@@ -139,7 +260,7 @@ class MemoryLlmWikiStore(LlmWikiStore):
row["reviewed_by"] = reviewer_user_id
row["published_at"] = _now() if approved else None
row["updated_at"] = _now()
return dict(row)
return _mapping_to_dict(row)
async def unpublish(self, mapping_id: str, actor_user_id: str, *, is_admin: bool) -> dict[str, Any] | None:
row = self._rows.get(mapping_id)
@@ -149,18 +270,18 @@ class MemoryLlmWikiStore(LlmWikiStore):
row["reviewed_by"] = actor_user_id if is_admin else row.get("reviewed_by")
row["published_at"] = None
row["updated_at"] = _now()
return dict(row)
return _mapping_to_dict(row)
async def list_publications(self, *, status: str | None = None) -> list[dict[str, Any]]:
rows = [row for row in self._rows.values() if row["publication_status"] != "private"]
if status:
rows = [row for row in rows if row["publication_status"] == status]
return [dict(row) for row in sorted(rows, key=lambda row: row["updated_at"], reverse=True)]
return [_mapping_to_dict(row) for row in sorted(rows, key=lambda row: row["updated_at"], reverse=True)]
async def get_conversation_deposit_mapping(self) -> dict[str, Any] | None:
for row in self._rows.values():
if row.get("owner_user_id") == "system" and row.get("name") == "对话沉淀" and row.get("publication_status") == "published":
return dict(row)
return _mapping_to_dict(row)
return None
async def get_conversation_deposit(
@@ -4,11 +4,11 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, String, UniqueConstraint, false, true
from sqlalchemy import Boolean, Integer, String, Text, UniqueConstraint, false, true
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
from deerflow.persistence.types import BeijingDateTime, PortableJSON
class LlmWikiKnowledgeBaseRow(Base):
@@ -19,6 +19,21 @@ class LlmWikiKnowledgeBaseRow(Base):
owner_user_id: Mapped[str] = mapped_column(String(64), index=True)
name: Mapped[str] = mapped_column(String(255), default="")
description: Mapped[str] = mapped_column(String(2048), default="")
# ``None`` means "use the product defaults derived from the knowledge-base
# name". An explicit empty list is meaningful: the owner disabled starter
# questions for this library.
test_questions: Mapped[list | None] = mapped_column(PortableJSON(), nullable=True)
retrieval_settings: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
wiki_parent_directories: Mapped[list[str] | None] = mapped_column(PortableJSON(), nullable=True)
wiki_directory_paths: Mapped[list[list[str]] | None] = mapped_column(PortableJSON(), nullable=True)
wiki_focus_instructions: Mapped[str | None] = mapped_column(Text, nullable=True)
wiki_content_instructions_raw: Mapped[str | None] = mapped_column(Text, nullable=True)
wiki_extraction_instructions_raw: Mapped[str | None] = mapped_column(Text, nullable=True)
wiki_generation_managed: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
wiki_config_schema_version: Mapped[int | None] = mapped_column(Integer, nullable=True)
wiki_config_revision: Mapped[int | None] = mapped_column(Integer, nullable=True)
wiki_config_apply_status: Mapped[str | None] = mapped_column(String(32), nullable=True)
wiki_config_apply_error: Mapped[str | None] = mapped_column(Text, nullable=True)
kb_type: Mapped[str] = mapped_column(String(32), default="document")
publication_status: Mapped[str] = mapped_column(String(16), default="private", index=True)
published_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
@@ -3,13 +3,19 @@
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from typing import Any, cast
from uuid import uuid4
from sqlalchemy import delete, or_, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.llmwiki.base import LlmWikiScope, LlmWikiStore
from deerflow.persistence.llmwiki.base import (
UNSET,
LlmWikiScope,
LlmWikiStore,
serialize_wiki_directory_paths,
serialize_wiki_parent_directories,
)
from deerflow.persistence.llmwiki.model import LlmWikiConversationDepositRow, LlmWikiKnowledgeBaseRow
@@ -19,6 +25,14 @@ def _now() -> datetime:
def _to_dict(row: LlmWikiKnowledgeBaseRow | LlmWikiConversationDepositRow) -> dict[str, Any]:
data = row.to_dict()
if "wiki_parent_directories" in data:
data["wiki_parent_directories"] = serialize_wiki_parent_directories(
data["wiki_parent_directories"]
)
if "wiki_directory_paths" in data:
data["wiki_directory_paths"] = serialize_wiki_directory_paths(
data["wiki_directory_paths"]
)
for key in ("published_at", "external_search_updated_at", "created_at", "updated_at"):
value = data.get(key)
if isinstance(value, datetime):
@@ -30,13 +44,51 @@ class LlmWikiRepository(LlmWikiStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def create_mapping(self, *, weknora_id: str, owner_user_id: str, name: str, description: str, kb_type: str) -> dict[str, Any]:
async def create_mapping(
self,
*,
weknora_id: str,
owner_user_id: str,
name: str,
description: str,
kb_type: str,
retrieval_settings: dict[str, Any] | None = None,
wiki_parent_directories: list[str] | None = None,
wiki_directory_paths: list[list[str]] | None = None,
wiki_focus_instructions: str | None = None,
wiki_content_instructions_raw: str | None = None,
wiki_extraction_instructions_raw: str | None = None,
wiki_generation_managed: bool | None = None,
wiki_config_schema_version: int | None = None,
wiki_config_revision: int | None = None,
wiki_config_apply_status: str | None = None,
wiki_config_apply_error: str | None = None,
) -> dict[str, Any]:
row = LlmWikiKnowledgeBaseRow(
id=str(uuid4()),
weknora_id=weknora_id,
owner_user_id=owner_user_id,
name=name,
description=description,
retrieval_settings=(
dict(retrieval_settings)
if retrieval_settings is not None
else None
),
wiki_parent_directories=serialize_wiki_parent_directories(
wiki_parent_directories
),
wiki_directory_paths=serialize_wiki_directory_paths(
wiki_directory_paths
),
wiki_focus_instructions=wiki_focus_instructions,
wiki_content_instructions_raw=wiki_content_instructions_raw,
wiki_extraction_instructions_raw=wiki_extraction_instructions_raw,
wiki_generation_managed=wiki_generation_managed,
wiki_config_schema_version=wiki_config_schema_version,
wiki_config_revision=wiki_config_revision,
wiki_config_apply_status=wiki_config_apply_status,
wiki_config_apply_error=wiki_config_apply_error,
kb_type=kb_type,
publication_status="private",
created_at=_now(),
@@ -88,8 +140,21 @@ class LlmWikiRepository(LlmWikiStore):
*,
name: str | None = None,
description: str | None = None,
test_questions: list[str] | None = None,
test_questions_set: bool = False,
retrieval_settings: dict[str, Any] | None = None,
wiki_index_enabled: bool | None = None,
external_search_enabled: bool | None = None,
wiki_parent_directories: list[str] | None | object = UNSET,
wiki_directory_paths: list[list[str]] | None | object = UNSET,
wiki_focus_instructions: str | None | object = UNSET,
wiki_content_instructions_raw: str | None | object = UNSET,
wiki_extraction_instructions_raw: str | None | object = UNSET,
wiki_generation_managed: bool | None | object = UNSET,
wiki_config_schema_version: int | None | object = UNSET,
wiki_config_revision: int | None | object = UNSET,
wiki_config_apply_status: str | None | object = UNSET,
wiki_config_apply_error: str | None | object = UNSET,
) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(LlmWikiKnowledgeBaseRow, mapping_id)
@@ -99,11 +164,63 @@ class LlmWikiRepository(LlmWikiStore):
row.name = name
if description is not None:
row.description = description
if test_questions_set:
row.test_questions = list(test_questions or [])
if retrieval_settings is not None:
row.retrieval_settings = dict(retrieval_settings)
if wiki_index_enabled is not None:
row.wiki_index_enabled = wiki_index_enabled
if external_search_enabled is not None:
row.external_search_enabled = external_search_enabled
row.external_search_updated_at = _now()
if wiki_parent_directories is not UNSET:
row.wiki_parent_directories = serialize_wiki_parent_directories(
cast(list[str] | None, wiki_parent_directories)
)
if wiki_directory_paths is not UNSET:
row.wiki_directory_paths = serialize_wiki_directory_paths(
cast(list[list[str]] | None, wiki_directory_paths)
)
if wiki_focus_instructions is not UNSET:
row.wiki_focus_instructions = cast(
str | None,
wiki_focus_instructions,
)
if wiki_content_instructions_raw is not UNSET:
row.wiki_content_instructions_raw = cast(
str | None,
wiki_content_instructions_raw,
)
if wiki_extraction_instructions_raw is not UNSET:
row.wiki_extraction_instructions_raw = cast(
str | None,
wiki_extraction_instructions_raw,
)
if wiki_generation_managed is not UNSET:
row.wiki_generation_managed = cast(
bool | None,
wiki_generation_managed,
)
if wiki_config_schema_version is not UNSET:
row.wiki_config_schema_version = cast(
int | None,
wiki_config_schema_version,
)
if wiki_config_revision is not UNSET:
row.wiki_config_revision = cast(
int | None,
wiki_config_revision,
)
if wiki_config_apply_status is not UNSET:
row.wiki_config_apply_status = cast(
str | None,
wiki_config_apply_status,
)
if wiki_config_apply_error is not UNSET:
row.wiki_config_apply_error = cast(
str | None,
wiki_config_apply_error,
)
row.updated_at = _now()
await session.commit()
return _to_dict(row)