sync update from LAN repo (source dhk-1 @ 3e87be55)

This commit is contained in:
gozeros committed 2026-10-21 15:06:08 +08:00
1 parent ab6068a7c1
commit 42f2a26fa8
1482 files changed
+227759 -9584

No files matched your search

@@ -0,0 +1,22 @@
"""Durable, owner-scoped records for the public-document workbench."""
from __future__ import annotations
from deerflow.persistence.gongwen_documents.memory import MemoryGongwenDocumentStore
from deerflow.persistence.gongwen_documents.model import GongwenDocumentRow, GongwenDocumentVersionRow
from deerflow.persistence.gongwen_documents.store import GongwenDocumentStore, SqlGongwenDocumentStore
__all__ = [
"GongwenDocumentRow",
"GongwenDocumentStore",
"GongwenDocumentVersionRow",
"MemoryGongwenDocumentStore",
"SqlGongwenDocumentStore",
"make_gongwen_document_store",
]
def make_gongwen_document_store(session_factory):
if session_factory is None:
return MemoryGongwenDocumentStore()
return SqlGongwenDocumentStore(session_factory)
@@ -0,0 +1,75 @@
"""Small in-memory fallback used when persistence is disabled in tests."""
from __future__ import annotations
from copy import deepcopy
from datetime import UTC, datetime
from typing import Any
from deerflow.persistence.gongwen_documents.store import GongwenDocumentStore
class MemoryGongwenDocumentStore(GongwenDocumentStore):
def __init__(self) -> None:
self._documents: dict[str, dict[str, Any]] = {}
self._versions: dict[str, list[dict[str, Any]]] = {}
@staticmethod
def _owned(document: dict[str, Any] | None, user_id: str | None) -> bool:
return bool(document and document.get("user_id") == user_id)
async def create_document(self, data: dict[str, Any]) -> dict[str, Any]:
now = datetime.now(UTC).isoformat()
row = {
"id": str(data["id"]), "user_id": data.get("user_id"), "title": str(data.get("title") or ""),
"document_type": str(data.get("document_type") or "notice"), "source": str(data.get("source") or "upload"),
"status": str(data.get("status") or "checking"), "template_key": data.get("template_key"),
"research_session_id": data.get("research_session_id"), "original_version_id": None, "latest_version_id": None,
"issue_counts": deepcopy(data.get("issue_counts")), "error": data.get("error"), "created_at": now, "updated_at": now,
}
self._documents[row["id"]] = row
return deepcopy(row)
async def list_documents(self, user_id: str | None, *, source: str | None = None, status: str | None = None, limit: int = 100) -> list[dict[str, Any]]:
rows = [r for r in self._documents.values() if r.get("user_id") == user_id]
if source:
rows = [r for r in rows if r.get("source") == source]
if status:
rows = [r for r in rows if r.get("status") == status]
return deepcopy(sorted(rows, key=lambda item: item["updated_at"], reverse=True)[:limit])
async def get_document(self, document_id: str, user_id: str | None) -> dict[str, Any] | None:
row = self._documents.get(document_id)
return deepcopy(row) if self._owned(row, user_id) else None
async def list_versions(self, document_id: str, user_id: str | None) -> list[dict[str, Any]]:
if not self._owned(self._documents.get(document_id), user_id):
return []
return deepcopy(list(reversed(self._versions.get(document_id, []))))
async def get_version(self, document_id: str, version_id: str, user_id: str | None) -> dict[str, Any] | None:
if not self._owned(self._documents.get(document_id), user_id):
return None
return next((deepcopy(row) for row in self._versions.get(document_id, []) if row["id"] == version_id), None)
async def append_version(self, document_id: str, user_id: str | None, version: dict[str, Any], document_patch: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any]] | None:
document = self._documents.get(document_id)
if not self._owned(document, user_id):
return None
rows = self._versions.setdefault(document_id, [])
row = {**deepcopy(version), "document_id": document_id, "version_no": len(rows) + 1, "created_at": datetime.now(UTC).isoformat()}
rows.append(row)
document.update(deepcopy(document_patch))
document["latest_version_id"] = row["id"]
if row["kind"] == "original" and not document.get("original_version_id"):
document["original_version_id"] = row["id"]
document["updated_at"] = datetime.now(UTC).isoformat()
return deepcopy(document), deepcopy(row)
async def update_document(self, document_id: str, user_id: str | None, patch: dict[str, Any]) -> dict[str, Any] | None:
document = self._documents.get(document_id)
if not self._owned(document, user_id):
return None
document.update(deepcopy(patch))
document["updated_at"] = datetime.now(UTC).isoformat()
return deepcopy(document)
@@ -0,0 +1,68 @@
"""ORM rows for the public-document library.
The binary files deliberately stay outside the database. A document row is
the durable library entry; every uploaded original, generated output and
format-repaired output is an immutable version row beneath it.
"""
from __future__ import annotations
from datetime import UTC, datetime
import sqlalchemy as sa
from sqlalchemy import Integer, String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableLongText
class GongwenDocumentRow(Base):
__tablename__ = "gongwen_documents"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
title: Mapped[str] = mapped_column(String(256), nullable=False, default="")
document_type: Mapped[str] = mapped_column(String(48), nullable=False, default="notice", index=True)
# upload | generated. Keep source separate from status so the library can
# filter both axes independently.
source: Mapped[str] = mapped_column(String(24), nullable=False, default="upload", index=True)
# checking | review_required | verified | repaired | failed
status: Mapped[str] = mapped_column(String(32), nullable=False, default="checking", index=True)
template_key: Mapped[str | None] = mapped_column(String(96), nullable=True)
research_session_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
original_version_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
latest_version_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
issue_counts: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
error: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
created_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
)
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)
)
class GongwenDocumentVersionRow(Base):
__tablename__ = "gongwen_document_versions"
__table_args__ = (
sa.UniqueConstraint("document_id", "version_no", name="uq_gongwen_document_version_no"),
sa.Index("ix_gongwen_document_versions_document_created", "document_id", "created_at"),
)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
document_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
version_no: Mapped[int] = mapped_column(Integer, nullable=False)
# original | generated | repaired | markdown
kind: Mapped[str] = mapped_column(String(32), nullable=False)
file_name: Mapped[str] = mapped_column(String(512), nullable=False)
storage_key: Mapped[str] = mapped_column(String(1024), nullable=False)
content_type: Mapped[str] = mapped_column(String(128), nullable=False, default="application/octet-stream")
size_bytes: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
sha256: Mapped[str] = mapped_column(String(64), nullable=False, default="")
# Exact review snapshot for this version. Later rule updates must not
# rewrite history or make past repair decisions impossible to explain.
review_snapshot: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
created_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
)
@@ -0,0 +1,203 @@
"""SQL and in-memory stores for public-document library metadata."""
from __future__ import annotations
import json
from abc import ABC, abstractmethod
from datetime import datetime
from typing import Any
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.gongwen_documents.model import GongwenDocumentRow, GongwenDocumentVersionRow
def _json_load(value: Any) -> Any:
if not isinstance(value, str) or not value:
return None
try:
return json.loads(value)
except json.JSONDecodeError:
return None
def _json_dump(value: Any) -> str | None:
if value is None:
return None
return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
def _as_dict(row: GongwenDocumentRow | GongwenDocumentVersionRow) -> dict[str, Any]:
result = row.to_dict()
for key, value in tuple(result.items()):
if isinstance(value, datetime):
result[key] = value.isoformat()
if "issue_counts" in result:
result["issue_counts"] = _json_load(result["issue_counts"])
if "review_snapshot" in result:
result["review_snapshot"] = _json_load(result["review_snapshot"])
return result
class GongwenDocumentStore(ABC):
@abstractmethod
async def create_document(self, data: dict[str, Any]) -> dict[str, Any]:
raise NotImplementedError
@abstractmethod
async def list_documents(
self, user_id: str | None, *, source: str | None = None, status: str | None = None, limit: int = 100
) -> list[dict[str, Any]]:
raise NotImplementedError
@abstractmethod
async def get_document(self, document_id: str, user_id: str | None) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def list_versions(self, document_id: str, user_id: str | None) -> list[dict[str, Any]]:
raise NotImplementedError
@abstractmethod
async def get_version(self, document_id: str, version_id: str, user_id: str | None) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def append_version(
self, document_id: str, user_id: str | None, version: dict[str, Any], document_patch: dict[str, Any]
) -> tuple[dict[str, Any], dict[str, Any]] | None:
raise NotImplementedError
@abstractmethod
async def update_document(self, document_id: str, user_id: str | None, patch: dict[str, Any]) -> dict[str, Any] | None:
raise NotImplementedError
class SqlGongwenDocumentStore(GongwenDocumentStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
@staticmethod
def _owned(stmt, user_id: str | None):
return stmt.where(GongwenDocumentRow.user_id == user_id)
async def create_document(self, data: dict[str, Any]) -> dict[str, Any]:
row = GongwenDocumentRow(
id=str(data["id"]),
user_id=data.get("user_id"),
title=str(data.get("title") or ""),
document_type=str(data.get("document_type") or "notice"),
source=str(data.get("source") or "upload"),
status=str(data.get("status") or "checking"),
template_key=data.get("template_key"),
research_session_id=data.get("research_session_id"),
issue_counts=_json_dump(data.get("issue_counts")),
error=data.get("error"),
)
async with self._sf() as session:
session.add(row)
await session.commit()
await session.refresh(row)
return _as_dict(row)
async def list_documents(
self, user_id: str | None, *, source: str | None = None, status: str | None = None, limit: int = 100
) -> list[dict[str, Any]]:
stmt = self._owned(select(GongwenDocumentRow), user_id)
if source:
stmt = stmt.where(GongwenDocumentRow.source == source)
if status:
stmt = stmt.where(GongwenDocumentRow.status == status)
stmt = stmt.order_by(GongwenDocumentRow.updated_at.desc()).limit(max(1, min(limit, 200)))
async with self._sf() as session:
return [_as_dict(row) for row in (await session.execute(stmt)).scalars()]
async def get_document(self, document_id: str, user_id: str | None) -> dict[str, Any] | None:
stmt = self._owned(select(GongwenDocumentRow).where(GongwenDocumentRow.id == document_id), user_id)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
return _as_dict(row) if row else None
async def list_versions(self, document_id: str, user_id: str | None) -> list[dict[str, Any]]:
stmt = self._owned(select(GongwenDocumentRow.id).where(GongwenDocumentRow.id == document_id), user_id)
async with self._sf() as session:
if (await session.execute(stmt)).scalar_one_or_none() is None:
return []
versions = await session.execute(
select(GongwenDocumentVersionRow)
.where(GongwenDocumentVersionRow.document_id == document_id)
.order_by(GongwenDocumentVersionRow.version_no.desc())
)
return [_as_dict(row) for row in versions.scalars()]
async def get_version(self, document_id: str, version_id: str, user_id: str | None) -> dict[str, Any] | None:
stmt = self._owned(select(GongwenDocumentRow.id).where(GongwenDocumentRow.id == document_id), user_id)
async with self._sf() as session:
if (await session.execute(stmt)).scalar_one_or_none() is None:
return None
row = (
await session.execute(
select(GongwenDocumentVersionRow).where(
GongwenDocumentVersionRow.document_id == document_id,
GongwenDocumentVersionRow.id == version_id,
)
)
).scalar_one_or_none()
return _as_dict(row) if row else None
async def append_version(
self, document_id: str, user_id: str | None, version: dict[str, Any], document_patch: dict[str, Any]
) -> tuple[dict[str, Any], dict[str, Any]] | None:
async with self._sf() as session:
stmt = self._owned(select(GongwenDocumentRow).where(GongwenDocumentRow.id == document_id), user_id)
document = (await session.execute(stmt)).scalar_one_or_none()
if document is None:
return None
version_no = int(
(await session.execute(
select(func.coalesce(func.max(GongwenDocumentVersionRow.version_no), 0)).where(
GongwenDocumentVersionRow.document_id == document_id
)
)).scalar_one()
) + 1
row = GongwenDocumentVersionRow(
id=str(version["id"]),
document_id=document_id,
version_no=version_no,
kind=str(version["kind"]),
file_name=str(version["file_name"]),
storage_key=str(version["storage_key"]),
content_type=str(version.get("content_type") or "application/octet-stream"),
size_bytes=int(version.get("size_bytes") or 0),
sha256=str(version.get("sha256") or ""),
review_snapshot=_json_dump(version.get("review_snapshot")),
)
for key, value in document_patch.items():
if key == "issue_counts":
value = _json_dump(value)
if hasattr(document, key):
setattr(document, key, value)
document.latest_version_id = row.id
if row.kind == "original" and not document.original_version_id:
document.original_version_id = row.id
session.add(row)
await session.commit()
await session.refresh(document)
await session.refresh(row)
return _as_dict(document), _as_dict(row)
async def update_document(self, document_id: str, user_id: str | None, patch: dict[str, Any]) -> dict[str, Any] | None:
async with self._sf() as session:
stmt = self._owned(select(GongwenDocumentRow).where(GongwenDocumentRow.id == document_id), user_id)
row = (await session.execute(stmt)).scalar_one_or_none()
if row is None:
return None
for key, value in patch.items():
if key == "issue_counts":
value = _json_dump(value)
if hasattr(row, key):
setattr(row, key, value)
await session.commit()
await session.refresh(row)
return _as_dict(row)