sync update from LAN repo (source dhk-1 @ 3e87be55)
This commit is contained in:
1 parent
ab6068a7c1
commit
42f2a26fa8
1482 files changed
+227759
-9584
No files matched your search
+22
@@ -0,0 +1,22 @@
|
||||
"""Durable, owner-scoped records for the public-document workbench."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.gongwen_documents.memory import MemoryGongwenDocumentStore
|
||||
from deerflow.persistence.gongwen_documents.model import GongwenDocumentRow, GongwenDocumentVersionRow
|
||||
from deerflow.persistence.gongwen_documents.store import GongwenDocumentStore, SqlGongwenDocumentStore
|
||||
|
||||
__all__ = [
|
||||
"GongwenDocumentRow",
|
||||
"GongwenDocumentStore",
|
||||
"GongwenDocumentVersionRow",
|
||||
"MemoryGongwenDocumentStore",
|
||||
"SqlGongwenDocumentStore",
|
||||
"make_gongwen_document_store",
|
||||
]
|
||||
|
||||
|
||||
def make_gongwen_document_store(session_factory):
|
||||
if session_factory is None:
|
||||
return MemoryGongwenDocumentStore()
|
||||
return SqlGongwenDocumentStore(session_factory)
|
||||
+75
@@ -0,0 +1,75 @@
|
||||
"""Small in-memory fallback used when persistence is disabled in tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from copy import deepcopy
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from deerflow.persistence.gongwen_documents.store import GongwenDocumentStore
|
||||
|
||||
|
||||
class MemoryGongwenDocumentStore(GongwenDocumentStore):
|
||||
def __init__(self) -> None:
|
||||
self._documents: dict[str, dict[str, Any]] = {}
|
||||
self._versions: dict[str, list[dict[str, Any]]] = {}
|
||||
|
||||
@staticmethod
|
||||
def _owned(document: dict[str, Any] | None, user_id: str | None) -> bool:
|
||||
return bool(document and document.get("user_id") == user_id)
|
||||
|
||||
async def create_document(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
now = datetime.now(UTC).isoformat()
|
||||
row = {
|
||||
"id": str(data["id"]), "user_id": data.get("user_id"), "title": str(data.get("title") or ""),
|
||||
"document_type": str(data.get("document_type") or "notice"), "source": str(data.get("source") or "upload"),
|
||||
"status": str(data.get("status") or "checking"), "template_key": data.get("template_key"),
|
||||
"research_session_id": data.get("research_session_id"), "original_version_id": None, "latest_version_id": None,
|
||||
"issue_counts": deepcopy(data.get("issue_counts")), "error": data.get("error"), "created_at": now, "updated_at": now,
|
||||
}
|
||||
self._documents[row["id"]] = row
|
||||
return deepcopy(row)
|
||||
|
||||
async def list_documents(self, user_id: str | None, *, source: str | None = None, status: str | None = None, limit: int = 100) -> list[dict[str, Any]]:
|
||||
rows = [r for r in self._documents.values() if r.get("user_id") == user_id]
|
||||
if source:
|
||||
rows = [r for r in rows if r.get("source") == source]
|
||||
if status:
|
||||
rows = [r for r in rows if r.get("status") == status]
|
||||
return deepcopy(sorted(rows, key=lambda item: item["updated_at"], reverse=True)[:limit])
|
||||
|
||||
async def get_document(self, document_id: str, user_id: str | None) -> dict[str, Any] | None:
|
||||
row = self._documents.get(document_id)
|
||||
return deepcopy(row) if self._owned(row, user_id) else None
|
||||
|
||||
async def list_versions(self, document_id: str, user_id: str | None) -> list[dict[str, Any]]:
|
||||
if not self._owned(self._documents.get(document_id), user_id):
|
||||
return []
|
||||
return deepcopy(list(reversed(self._versions.get(document_id, []))))
|
||||
|
||||
async def get_version(self, document_id: str, version_id: str, user_id: str | None) -> dict[str, Any] | None:
|
||||
if not self._owned(self._documents.get(document_id), user_id):
|
||||
return None
|
||||
return next((deepcopy(row) for row in self._versions.get(document_id, []) if row["id"] == version_id), None)
|
||||
|
||||
async def append_version(self, document_id: str, user_id: str | None, version: dict[str, Any], document_patch: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any]] | None:
|
||||
document = self._documents.get(document_id)
|
||||
if not self._owned(document, user_id):
|
||||
return None
|
||||
rows = self._versions.setdefault(document_id, [])
|
||||
row = {**deepcopy(version), "document_id": document_id, "version_no": len(rows) + 1, "created_at": datetime.now(UTC).isoformat()}
|
||||
rows.append(row)
|
||||
document.update(deepcopy(document_patch))
|
||||
document["latest_version_id"] = row["id"]
|
||||
if row["kind"] == "original" and not document.get("original_version_id"):
|
||||
document["original_version_id"] = row["id"]
|
||||
document["updated_at"] = datetime.now(UTC).isoformat()
|
||||
return deepcopy(document), deepcopy(row)
|
||||
|
||||
async def update_document(self, document_id: str, user_id: str | None, patch: dict[str, Any]) -> dict[str, Any] | None:
|
||||
document = self._documents.get(document_id)
|
||||
if not self._owned(document, user_id):
|
||||
return None
|
||||
document.update(deepcopy(patch))
|
||||
document["updated_at"] = datetime.now(UTC).isoformat()
|
||||
return deepcopy(document)
|
||||
+68
@@ -0,0 +1,68 @@
|
||||
"""ORM rows for the public-document library.
|
||||
|
||||
The binary files deliberately stay outside the database. A document row is
|
||||
the durable library entry; every uploaded original, generated output and
|
||||
format-repaired output is an immutable version row beneath it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy import Integer, String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableLongText
|
||||
|
||||
|
||||
class GongwenDocumentRow(Base):
|
||||
__tablename__ = "gongwen_documents"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
title: Mapped[str] = mapped_column(String(256), nullable=False, default="")
|
||||
document_type: Mapped[str] = mapped_column(String(48), nullable=False, default="notice", index=True)
|
||||
# upload | generated. Keep source separate from status so the library can
|
||||
# filter both axes independently.
|
||||
source: Mapped[str] = mapped_column(String(24), nullable=False, default="upload", index=True)
|
||||
# checking | review_required | verified | repaired | failed
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False, default="checking", index=True)
|
||||
template_key: Mapped[str | None] = mapped_column(String(96), nullable=True)
|
||||
research_session_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
original_version_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
latest_version_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
issue_counts: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
error: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)
|
||||
)
|
||||
|
||||
|
||||
class GongwenDocumentVersionRow(Base):
|
||||
__tablename__ = "gongwen_document_versions"
|
||||
__table_args__ = (
|
||||
sa.UniqueConstraint("document_id", "version_no", name="uq_gongwen_document_version_no"),
|
||||
sa.Index("ix_gongwen_document_versions_document_created", "document_id", "created_at"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
document_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
version_no: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
# original | generated | repaired | markdown
|
||||
kind: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
file_name: Mapped[str] = mapped_column(String(512), nullable=False)
|
||||
storage_key: Mapped[str] = mapped_column(String(1024), nullable=False)
|
||||
content_type: Mapped[str] = mapped_column(String(128), nullable=False, default="application/octet-stream")
|
||||
size_bytes: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
sha256: Mapped[str] = mapped_column(String(64), nullable=False, default="")
|
||||
# Exact review snapshot for this version. Later rule updates must not
|
||||
# rewrite history or make past repair decisions impossible to explain.
|
||||
review_snapshot: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
|
||||
)
|
||||
+203
@@ -0,0 +1,203 @@
|
||||
"""SQL and in-memory stores for public-document library metadata."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from abc import ABC, abstractmethod
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.gongwen_documents.model import GongwenDocumentRow, GongwenDocumentVersionRow
|
||||
|
||||
|
||||
def _json_load(value: Any) -> Any:
|
||||
if not isinstance(value, str) or not value:
|
||||
return None
|
||||
try:
|
||||
return json.loads(value)
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
|
||||
|
||||
def _json_dump(value: Any) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
|
||||
|
||||
|
||||
def _as_dict(row: GongwenDocumentRow | GongwenDocumentVersionRow) -> dict[str, Any]:
|
||||
result = row.to_dict()
|
||||
for key, value in tuple(result.items()):
|
||||
if isinstance(value, datetime):
|
||||
result[key] = value.isoformat()
|
||||
if "issue_counts" in result:
|
||||
result["issue_counts"] = _json_load(result["issue_counts"])
|
||||
if "review_snapshot" in result:
|
||||
result["review_snapshot"] = _json_load(result["review_snapshot"])
|
||||
return result
|
||||
|
||||
|
||||
class GongwenDocumentStore(ABC):
|
||||
@abstractmethod
|
||||
async def create_document(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_documents(
|
||||
self, user_id: str | None, *, source: str | None = None, status: str | None = None, limit: int = 100
|
||||
) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_document(self, document_id: str, user_id: str | None) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_versions(self, document_id: str, user_id: str | None) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_version(self, document_id: str, version_id: str, user_id: str | None) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def append_version(
|
||||
self, document_id: str, user_id: str | None, version: dict[str, Any], document_patch: dict[str, Any]
|
||||
) -> tuple[dict[str, Any], dict[str, Any]] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def update_document(self, document_id: str, user_id: str | None, patch: dict[str, Any]) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class SqlGongwenDocumentStore(GongwenDocumentStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
@staticmethod
|
||||
def _owned(stmt, user_id: str | None):
|
||||
return stmt.where(GongwenDocumentRow.user_id == user_id)
|
||||
|
||||
async def create_document(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
row = GongwenDocumentRow(
|
||||
id=str(data["id"]),
|
||||
user_id=data.get("user_id"),
|
||||
title=str(data.get("title") or ""),
|
||||
document_type=str(data.get("document_type") or "notice"),
|
||||
source=str(data.get("source") or "upload"),
|
||||
status=str(data.get("status") or "checking"),
|
||||
template_key=data.get("template_key"),
|
||||
research_session_id=data.get("research_session_id"),
|
||||
issue_counts=_json_dump(data.get("issue_counts")),
|
||||
error=data.get("error"),
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return _as_dict(row)
|
||||
|
||||
async def list_documents(
|
||||
self, user_id: str | None, *, source: str | None = None, status: str | None = None, limit: int = 100
|
||||
) -> list[dict[str, Any]]:
|
||||
stmt = self._owned(select(GongwenDocumentRow), user_id)
|
||||
if source:
|
||||
stmt = stmt.where(GongwenDocumentRow.source == source)
|
||||
if status:
|
||||
stmt = stmt.where(GongwenDocumentRow.status == status)
|
||||
stmt = stmt.order_by(GongwenDocumentRow.updated_at.desc()).limit(max(1, min(limit, 200)))
|
||||
async with self._sf() as session:
|
||||
return [_as_dict(row) for row in (await session.execute(stmt)).scalars()]
|
||||
|
||||
async def get_document(self, document_id: str, user_id: str | None) -> dict[str, Any] | None:
|
||||
stmt = self._owned(select(GongwenDocumentRow).where(GongwenDocumentRow.id == document_id), user_id)
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
return _as_dict(row) if row else None
|
||||
|
||||
async def list_versions(self, document_id: str, user_id: str | None) -> list[dict[str, Any]]:
|
||||
stmt = self._owned(select(GongwenDocumentRow.id).where(GongwenDocumentRow.id == document_id), user_id)
|
||||
async with self._sf() as session:
|
||||
if (await session.execute(stmt)).scalar_one_or_none() is None:
|
||||
return []
|
||||
versions = await session.execute(
|
||||
select(GongwenDocumentVersionRow)
|
||||
.where(GongwenDocumentVersionRow.document_id == document_id)
|
||||
.order_by(GongwenDocumentVersionRow.version_no.desc())
|
||||
)
|
||||
return [_as_dict(row) for row in versions.scalars()]
|
||||
|
||||
async def get_version(self, document_id: str, version_id: str, user_id: str | None) -> dict[str, Any] | None:
|
||||
stmt = self._owned(select(GongwenDocumentRow.id).where(GongwenDocumentRow.id == document_id), user_id)
|
||||
async with self._sf() as session:
|
||||
if (await session.execute(stmt)).scalar_one_or_none() is None:
|
||||
return None
|
||||
row = (
|
||||
await session.execute(
|
||||
select(GongwenDocumentVersionRow).where(
|
||||
GongwenDocumentVersionRow.document_id == document_id,
|
||||
GongwenDocumentVersionRow.id == version_id,
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
return _as_dict(row) if row else None
|
||||
|
||||
async def append_version(
|
||||
self, document_id: str, user_id: str | None, version: dict[str, Any], document_patch: dict[str, Any]
|
||||
) -> tuple[dict[str, Any], dict[str, Any]] | None:
|
||||
async with self._sf() as session:
|
||||
stmt = self._owned(select(GongwenDocumentRow).where(GongwenDocumentRow.id == document_id), user_id)
|
||||
document = (await session.execute(stmt)).scalar_one_or_none()
|
||||
if document is None:
|
||||
return None
|
||||
version_no = int(
|
||||
(await session.execute(
|
||||
select(func.coalesce(func.max(GongwenDocumentVersionRow.version_no), 0)).where(
|
||||
GongwenDocumentVersionRow.document_id == document_id
|
||||
)
|
||||
)).scalar_one()
|
||||
) + 1
|
||||
row = GongwenDocumentVersionRow(
|
||||
id=str(version["id"]),
|
||||
document_id=document_id,
|
||||
version_no=version_no,
|
||||
kind=str(version["kind"]),
|
||||
file_name=str(version["file_name"]),
|
||||
storage_key=str(version["storage_key"]),
|
||||
content_type=str(version.get("content_type") or "application/octet-stream"),
|
||||
size_bytes=int(version.get("size_bytes") or 0),
|
||||
sha256=str(version.get("sha256") or ""),
|
||||
review_snapshot=_json_dump(version.get("review_snapshot")),
|
||||
)
|
||||
for key, value in document_patch.items():
|
||||
if key == "issue_counts":
|
||||
value = _json_dump(value)
|
||||
if hasattr(document, key):
|
||||
setattr(document, key, value)
|
||||
document.latest_version_id = row.id
|
||||
if row.kind == "original" and not document.original_version_id:
|
||||
document.original_version_id = row.id
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(document)
|
||||
await session.refresh(row)
|
||||
return _as_dict(document), _as_dict(row)
|
||||
|
||||
async def update_document(self, document_id: str, user_id: str | None, patch: dict[str, Any]) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
stmt = self._owned(select(GongwenDocumentRow).where(GongwenDocumentRow.id == document_id), user_id)
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
if row is None:
|
||||
return None
|
||||
for key, value in patch.items():
|
||||
if key == "issue_counts":
|
||||
value = _json_dump(value)
|
||||
if hasattr(row, key):
|
||||
setattr(row, key, value)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return _as_dict(row)
|
||||
Reference in new issue
Block a user