初始化

This commit is contained in:
gozeros committed 2026-09-07 18:24:55 +08:00
1 parent 34474bd5fc
commit b638b541d0
3008 files changed
+837699

No files matched your search

@@ -0,0 +1,22 @@
"""DeerFlow-owned LLMWiki authorization metadata."""
from deerflow.persistence.llmwiki.base import LlmWikiScope, LlmWikiStore
from deerflow.persistence.llmwiki.memory import MemoryLlmWikiStore
from deerflow.persistence.llmwiki.model import LlmWikiConversationDepositRow, LlmWikiKnowledgeBaseRow
from deerflow.persistence.llmwiki.sql import LlmWikiRepository
__all__ = [
"LlmWikiKnowledgeBaseRow",
"LlmWikiConversationDepositRow",
"LlmWikiRepository",
"LlmWikiScope",
"LlmWikiStore",
"MemoryLlmWikiStore",
"make_llmwiki_store",
]
def make_llmwiki_store(session_factory):
if session_factory is None:
return MemoryLlmWikiStore()
return LlmWikiRepository(session_factory)
@@ -0,0 +1,89 @@
"""Persistence contract for DeerFlow-owned LLMWiki authorization metadata."""
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any, Literal
LlmWikiScope = Literal["personal", "public", "all"]
class LlmWikiStore(ABC):
@abstractmethod
async def create_mapping(self, *, weknora_id: str, owner_user_id: str, name: str, description: str, kb_type: str) -> dict[str, Any]: ...
@abstractmethod
async def list_visible(self, user_id: str, *, scope: LlmWikiScope = "all", is_admin: bool = False) -> list[dict[str, Any]]: ...
@abstractmethod
async def get_authorized(self, mapping_id: str, user_id: str, *, write: bool, is_admin: bool) -> dict[str, Any] | None: ...
@abstractmethod
async def get_by_weknora_id(self, weknora_id: str) -> dict[str, Any] | None: ...
@abstractmethod
async def update_mapping(
self,
mapping_id: str,
*,
name: str | None = None,
description: str | None = None,
wiki_index_enabled: bool | None = None,
external_search_enabled: bool | None = None,
) -> dict[str, Any] | None: ...
@abstractmethod
async def adopt_conversation_deposit_mapping(
self,
mapping_id: str,
*,
name: str,
description: str,
) -> dict[str, Any] | None: ...
@abstractmethod
async def delete_mapping(self, mapping_id: str) -> bool: ...
@abstractmethod
async def request_publish(self, mapping_id: str, actor_user_id: str, *, is_admin: bool) -> dict[str, Any] | None: ...
@abstractmethod
async def review_publish(self, mapping_id: str, reviewer_user_id: str, *, approved: bool) -> dict[str, Any] | None: ...
@abstractmethod
async def unpublish(self, mapping_id: str, actor_user_id: str, *, is_admin: bool) -> dict[str, Any] | None: ...
@abstractmethod
async def list_publications(self, *, status: str | None = None) -> list[dict[str, Any]]: ...
@abstractmethod
async def get_conversation_deposit_mapping(self) -> dict[str, Any] | None: ...
@abstractmethod
async def get_conversation_deposit(
self,
*,
thread_id: str,
assistant_message_id: str,
) -> dict[str, Any] | None: ...
@abstractmethod
async def upsert_conversation_deposit(
self,
*,
thread_id: str,
human_message_id: str | None,
assistant_message_id: str,
knowledge_base_mapping_id: str,
weknora_knowledge_id: str | None,
title: str,
created_by_user_id: str | None,
) -> dict[str, Any]: ...
@abstractmethod
async def delete_conversation_deposits(
self,
*,
thread_id: str,
message_ids: list[str] | None = None,
) -> list[dict[str, Any]]: ...
@@ -0,0 +1,231 @@
"""In-memory LLMWiki metadata store for tests and DB-less deployments."""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from uuid import uuid4
from deerflow.persistence.llmwiki.base import LlmWikiScope, LlmWikiStore
def _now() -> str:
return datetime.now(UTC).isoformat()
class MemoryLlmWikiStore(LlmWikiStore):
def __init__(self) -> None:
self._rows: dict[str, dict[str, Any]] = {}
self._conversation_deposits: dict[str, dict[str, Any]] = {}
async def create_mapping(self, *, weknora_id: str, owner_user_id: str, name: str, description: str, kb_type: str) -> dict[str, Any]:
if any(row["weknora_id"] == weknora_id for row in self._rows.values()):
raise ValueError("WeKnora knowledge base is already mapped")
now = _now()
row = {
"id": str(uuid4()),
"weknora_id": weknora_id,
"owner_user_id": owner_user_id,
"name": name,
"description": description,
"kb_type": kb_type,
"publication_status": "private",
"published_by": None,
"reviewed_by": None,
"published_at": None,
"wiki_index_enabled": True,
"external_search_enabled": False,
"external_search_updated_at": None,
"created_at": now,
"updated_at": now,
}
self._rows[row["id"]] = row
return dict(row)
async def list_visible(self, user_id: str, *, scope: LlmWikiScope = "all", is_admin: bool = False) -> list[dict[str, Any]]:
rows = list(self._rows.values())
if scope == "personal":
rows = rows if is_admin else [row for row in rows if row["owner_user_id"] == user_id]
elif scope == "public":
rows = [row for row in rows if row["publication_status"] == "published"]
else:
rows = rows if is_admin else [row for row in rows if row["owner_user_id"] == user_id or row["publication_status"] == "published"]
return [dict(row) for row in sorted(rows, key=lambda row: row["created_at"], reverse=True)]
async def get_authorized(self, mapping_id: str, user_id: str, *, write: bool, is_admin: bool) -> dict[str, Any] | None:
row = self._rows.get(mapping_id)
if row is None:
return None
if is_admin or row["owner_user_id"] == user_id:
return dict(row)
if not write and row["publication_status"] == "published":
return dict(row)
return None
async def get_by_weknora_id(self, weknora_id: str) -> dict[str, Any] | None:
for row in self._rows.values():
if row["weknora_id"] == weknora_id:
return dict(row)
return None
async def update_mapping(
self,
mapping_id: str,
*,
name: str | None = None,
description: str | None = None,
wiki_index_enabled: bool | None = None,
external_search_enabled: bool | None = None,
) -> dict[str, Any] | None:
row = self._rows.get(mapping_id)
if row is None:
return None
if name is not None:
row["name"] = name
if description is not None:
row["description"] = description
if wiki_index_enabled is not None:
row["wiki_index_enabled"] = wiki_index_enabled
if external_search_enabled is not None:
row["external_search_enabled"] = external_search_enabled
row["external_search_updated_at"] = _now()
row["updated_at"] = _now()
return dict(row)
async def adopt_conversation_deposit_mapping(
self,
mapping_id: str,
*,
name: str,
description: str,
) -> dict[str, Any] | None:
row = self._rows.get(mapping_id)
if row is None:
return None
now = _now()
row.update(
{
"owner_user_id": "system",
"name": name,
"description": description,
"publication_status": "published",
"published_by": "system",
"reviewed_by": None,
"published_at": row.get("published_at") or now,
"updated_at": now,
}
)
return dict(row)
async def delete_mapping(self, mapping_id: str) -> bool:
return self._rows.pop(mapping_id, None) is not None
async def request_publish(self, mapping_id: str, actor_user_id: str, *, is_admin: bool) -> dict[str, Any] | None:
row = self._rows.get(mapping_id)
if row is None or (not is_admin and row["owner_user_id"] != actor_user_id):
return None
row["publication_status"] = "published"
row["published_by"] = actor_user_id
row["reviewed_by"] = None
row["published_at"] = _now()
row["updated_at"] = _now()
return dict(row)
async def review_publish(self, mapping_id: str, reviewer_user_id: str, *, approved: bool) -> dict[str, Any] | None:
row = self._rows.get(mapping_id)
if row is None or row["publication_status"] != "pending":
return None
row["publication_status"] = "published" if approved else "rejected"
row["reviewed_by"] = reviewer_user_id
row["published_at"] = _now() if approved else None
row["updated_at"] = _now()
return dict(row)
async def unpublish(self, mapping_id: str, actor_user_id: str, *, is_admin: bool) -> dict[str, Any] | None:
row = self._rows.get(mapping_id)
if row is None or (not is_admin and row["owner_user_id"] != actor_user_id):
return None
row["publication_status"] = "private"
row["reviewed_by"] = actor_user_id if is_admin else row.get("reviewed_by")
row["published_at"] = None
row["updated_at"] = _now()
return dict(row)
async def list_publications(self, *, status: str | None = None) -> list[dict[str, Any]]:
rows = [row for row in self._rows.values() if row["publication_status"] != "private"]
if status:
rows = [row for row in rows if row["publication_status"] == status]
return [dict(row) for row in sorted(rows, key=lambda row: row["updated_at"], reverse=True)]
async def get_conversation_deposit_mapping(self) -> dict[str, Any] | None:
for row in self._rows.values():
if row.get("owner_user_id") == "system" and row.get("name") == "对话沉淀" and row.get("publication_status") == "published":
return dict(row)
return None
async def get_conversation_deposit(
self,
*,
thread_id: str,
assistant_message_id: str,
) -> dict[str, Any] | None:
key = f"{thread_id}:{assistant_message_id}"
row = self._conversation_deposits.get(key)
return dict(row) if row else None
async def upsert_conversation_deposit(
self,
*,
thread_id: str,
human_message_id: str | None,
assistant_message_id: str,
knowledge_base_mapping_id: str,
weknora_knowledge_id: str | None,
title: str,
created_by_user_id: str | None,
) -> dict[str, Any]:
key = f"{thread_id}:{assistant_message_id}"
now = _now()
row = self._conversation_deposits.get(key)
if row is None:
row = {
"id": str(uuid4()),
"thread_id": thread_id,
"human_message_id": human_message_id,
"assistant_message_id": assistant_message_id,
"knowledge_base_mapping_id": knowledge_base_mapping_id,
"weknora_knowledge_id": weknora_knowledge_id,
"title": title,
"created_by_user_id": created_by_user_id,
"created_at": now,
"updated_at": now,
}
self._conversation_deposits[key] = row
else:
row.update(
{
"human_message_id": human_message_id,
"knowledge_base_mapping_id": knowledge_base_mapping_id,
"weknora_knowledge_id": weknora_knowledge_id,
"title": title,
"updated_at": now,
}
)
return dict(row)
async def delete_conversation_deposits(
self,
*,
thread_id: str,
message_ids: list[str] | None = None,
) -> list[dict[str, Any]]:
ids = set(message_ids or [])
deleted: list[dict[str, Any]] = []
for key, row in list(self._conversation_deposits.items()):
if row.get("thread_id") != thread_id:
continue
if ids and row.get("human_message_id") not in ids and row.get("assistant_message_id") not in ids:
continue
deleted.append(dict(row))
self._conversation_deposits.pop(key, None)
return deleted
@@ -0,0 +1,66 @@
"""ORM mapping between DeerFlow ownership and WeKnora knowledge bases."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, String, UniqueConstraint, false, true
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class LlmWikiKnowledgeBaseRow(Base):
__tablename__ = "llmwiki_knowledge_bases"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
weknora_id: Mapped[str] = mapped_column(String(128), unique=True, index=True)
owner_user_id: Mapped[str] = mapped_column(String(64), index=True)
name: Mapped[str] = mapped_column(String(255), default="")
description: Mapped[str] = mapped_column(String(2048), default="")
kb_type: Mapped[str] = mapped_column(String(32), default="document")
publication_status: Mapped[str] = mapped_column(String(16), default="private", index=True)
published_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
reviewed_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
published_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
wiki_index_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default=true(), index=True)
external_search_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default=false(), index=True)
external_search_updated_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
def to_dict(self) -> dict:
return {column.name: getattr(self, column.name) for column in self.__table__.columns}
class LlmWikiConversationDepositRow(Base):
"""Mapping from one DeerFlow answer turn to its WeKnora manual document.
The actual content lives in WeKnora. DeerFlow keeps only the stable ids it
needs to rewrite/delete the document when the user regenerates or removes a
conversation turn.
"""
__tablename__ = "llmwiki_conversation_deposits"
__table_args__ = (
UniqueConstraint(
"thread_id",
"assistant_message_id",
name="uq_llmwiki_conversation_deposit_answer",
),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
thread_id: Mapped[str] = mapped_column(String(128), index=True)
human_message_id: Mapped[str | None] = mapped_column(String(128), nullable=True, index=True)
assistant_message_id: Mapped[str] = mapped_column(String(128), index=True)
knowledge_base_mapping_id: Mapped[str] = mapped_column(String(36), index=True)
weknora_knowledge_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
title: Mapped[str] = mapped_column(String(255), default="")
created_by_user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
def to_dict(self) -> dict:
return {column.name: getattr(self, column.name) for column in self.__table__.columns}
@@ -0,0 +1,281 @@
"""SQLAlchemy LLMWiki metadata repository."""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from uuid import uuid4
from sqlalchemy import delete, or_, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.llmwiki.base import LlmWikiScope, LlmWikiStore
from deerflow.persistence.llmwiki.model import LlmWikiConversationDepositRow, LlmWikiKnowledgeBaseRow
def _now() -> datetime:
return datetime.now(UTC)
def _to_dict(row: LlmWikiKnowledgeBaseRow | LlmWikiConversationDepositRow) -> dict[str, Any]:
data = row.to_dict()
for key in ("published_at", "external_search_updated_at", "created_at", "updated_at"):
value = data.get(key)
if isinstance(value, datetime):
data[key] = value.isoformat()
return data
class LlmWikiRepository(LlmWikiStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def create_mapping(self, *, weknora_id: str, owner_user_id: str, name: str, description: str, kb_type: str) -> dict[str, Any]:
row = LlmWikiKnowledgeBaseRow(
id=str(uuid4()),
weknora_id=weknora_id,
owner_user_id=owner_user_id,
name=name,
description=description,
kb_type=kb_type,
publication_status="private",
created_at=_now(),
updated_at=_now(),
)
async with self._sf() as session:
session.add(row)
await session.commit()
return _to_dict(row)
async def list_visible(self, user_id: str, *, scope: LlmWikiScope = "all", is_admin: bool = False) -> list[dict[str, Any]]:
stmt = select(LlmWikiKnowledgeBaseRow)
if scope == "personal" and not is_admin:
stmt = stmt.where(LlmWikiKnowledgeBaseRow.owner_user_id == user_id)
elif scope == "public":
stmt = stmt.where(LlmWikiKnowledgeBaseRow.publication_status == "published")
elif scope == "all" and not is_admin:
stmt = stmt.where(
or_(
LlmWikiKnowledgeBaseRow.owner_user_id == user_id,
LlmWikiKnowledgeBaseRow.publication_status == "published",
)
)
stmt = stmt.order_by(LlmWikiKnowledgeBaseRow.created_at.desc())
async with self._sf() as session:
result = await session.execute(stmt)
return [_to_dict(row) for row in result.scalars()]
async def get_authorized(self, mapping_id: str, user_id: str, *, write: bool, is_admin: bool) -> dict[str, Any] | None:
stmt = select(LlmWikiKnowledgeBaseRow).where(LlmWikiKnowledgeBaseRow.id == mapping_id)
if not is_admin:
condition = LlmWikiKnowledgeBaseRow.owner_user_id == user_id
if not write:
condition = or_(condition, LlmWikiKnowledgeBaseRow.publication_status == "published")
stmt = stmt.where(condition)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
return _to_dict(row) if row else None
async def get_by_weknora_id(self, weknora_id: str) -> dict[str, Any] | None:
stmt = select(LlmWikiKnowledgeBaseRow).where(LlmWikiKnowledgeBaseRow.weknora_id == weknora_id)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
return _to_dict(row) if row else None
async def update_mapping(
self,
mapping_id: str,
*,
name: str | None = None,
description: str | None = None,
wiki_index_enabled: bool | None = None,
external_search_enabled: bool | None = None,
) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(LlmWikiKnowledgeBaseRow, mapping_id)
if row is None:
return None
if name is not None:
row.name = name
if description is not None:
row.description = description
if wiki_index_enabled is not None:
row.wiki_index_enabled = wiki_index_enabled
if external_search_enabled is not None:
row.external_search_enabled = external_search_enabled
row.external_search_updated_at = _now()
row.updated_at = _now()
await session.commit()
return _to_dict(row)
async def adopt_conversation_deposit_mapping(
self,
mapping_id: str,
*,
name: str,
description: str,
) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(LlmWikiKnowledgeBaseRow, mapping_id)
if row is None:
return None
now = _now()
row.owner_user_id = "system"
row.name = name
row.description = description
row.publication_status = "published"
row.published_by = "system"
row.reviewed_by = None
row.published_at = row.published_at or now
row.updated_at = now
await session.commit()
return _to_dict(row)
async def delete_mapping(self, mapping_id: str) -> bool:
async with self._sf() as session:
row = await session.get(LlmWikiKnowledgeBaseRow, mapping_id)
if row is None:
return False
await session.delete(row)
await session.commit()
return True
async def request_publish(self, mapping_id: str, actor_user_id: str, *, is_admin: bool) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(LlmWikiKnowledgeBaseRow, mapping_id)
if row is None or (not is_admin and row.owner_user_id != actor_user_id):
return None
row.publication_status = "published"
row.published_by = actor_user_id
row.reviewed_by = None
row.published_at = _now()
row.updated_at = _now()
await session.commit()
return _to_dict(row)
async def review_publish(self, mapping_id: str, reviewer_user_id: str, *, approved: bool) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(LlmWikiKnowledgeBaseRow, mapping_id)
if row is None or row.publication_status != "pending":
return None
row.publication_status = "published" if approved else "rejected"
row.reviewed_by = reviewer_user_id
row.published_at = _now() if approved else None
row.updated_at = _now()
await session.commit()
return _to_dict(row)
async def unpublish(self, mapping_id: str, actor_user_id: str, *, is_admin: bool) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(LlmWikiKnowledgeBaseRow, mapping_id)
if row is None or (not is_admin and row.owner_user_id != actor_user_id):
return None
row.publication_status = "private"
if is_admin:
row.reviewed_by = actor_user_id
row.published_at = None
row.updated_at = _now()
await session.commit()
return _to_dict(row)
async def list_publications(self, *, status: str | None = None) -> list[dict[str, Any]]:
stmt = select(LlmWikiKnowledgeBaseRow).where(LlmWikiKnowledgeBaseRow.publication_status != "private")
if status:
stmt = stmt.where(LlmWikiKnowledgeBaseRow.publication_status == status)
stmt = stmt.order_by(LlmWikiKnowledgeBaseRow.updated_at.desc())
async with self._sf() as session:
result = await session.execute(stmt)
return [_to_dict(row) for row in result.scalars()]
async def get_conversation_deposit_mapping(self) -> dict[str, Any] | None:
stmt = (
select(LlmWikiKnowledgeBaseRow)
.where(
LlmWikiKnowledgeBaseRow.owner_user_id == "system",
LlmWikiKnowledgeBaseRow.name == "对话沉淀",
LlmWikiKnowledgeBaseRow.publication_status == "published",
)
.order_by(LlmWikiKnowledgeBaseRow.created_at.asc())
.limit(1)
)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
return _to_dict(row) if row else None
async def get_conversation_deposit(
self,
*,
thread_id: str,
assistant_message_id: str,
) -> dict[str, Any] | None:
stmt = select(LlmWikiConversationDepositRow).where(
LlmWikiConversationDepositRow.thread_id == thread_id,
LlmWikiConversationDepositRow.assistant_message_id == assistant_message_id,
)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
return _to_dict(row) if row else None
async def upsert_conversation_deposit(
self,
*,
thread_id: str,
human_message_id: str | None,
assistant_message_id: str,
knowledge_base_mapping_id: str,
weknora_knowledge_id: str | None,
title: str,
created_by_user_id: str | None,
) -> dict[str, Any]:
now = _now()
async with self._sf() as session:
stmt = select(LlmWikiConversationDepositRow).where(
LlmWikiConversationDepositRow.thread_id == thread_id,
LlmWikiConversationDepositRow.assistant_message_id == assistant_message_id,
)
row = (await session.execute(stmt)).scalar_one_or_none()
if row is None:
row = LlmWikiConversationDepositRow(
id=str(uuid4()),
thread_id=thread_id,
human_message_id=human_message_id,
assistant_message_id=assistant_message_id,
knowledge_base_mapping_id=knowledge_base_mapping_id,
weknora_knowledge_id=weknora_knowledge_id,
title=title,
created_by_user_id=created_by_user_id,
created_at=now,
updated_at=now,
)
session.add(row)
else:
row.human_message_id = human_message_id
row.knowledge_base_mapping_id = knowledge_base_mapping_id
row.weknora_knowledge_id = weknora_knowledge_id
row.title = title
row.updated_at = now
await session.commit()
return _to_dict(row)
async def delete_conversation_deposits(
self,
*,
thread_id: str,
message_ids: list[str] | None = None,
) -> list[dict[str, Any]]:
conditions = [LlmWikiConversationDepositRow.thread_id == thread_id]
if message_ids:
conditions.append(
or_(
LlmWikiConversationDepositRow.human_message_id.in_(message_ids),
LlmWikiConversationDepositRow.assistant_message_id.in_(message_ids),
)
)
stmt = select(LlmWikiConversationDepositRow).where(*conditions)
async with self._sf() as session:
rows = (await session.execute(stmt)).scalars().all()
deleted = [_to_dict(row) for row in rows]
if rows:
await session.execute(delete(LlmWikiConversationDepositRow).where(LlmWikiConversationDepositRow.id.in_([row.id for row in rows])))
await session.commit()
return deleted