初始化

This commit is contained in:
gozeros committed 2026-09-07 18:24:55 +08:00
1 parent 34474bd5fc
commit b638b541d0
3008 files changed
+837699

No files matched your search

@@ -0,0 +1,33 @@
"""Deep-research source-material persistence."""
from __future__ import annotations
from deerflow.persistence.deep_research_sources.model import DeepResearchSourceRow
from deerflow.persistence.deep_research_sources.sql import DeepResearchSourceRepository
def make_deep_research_source_store(session_factory) -> DeepResearchSourceRepository | None:
if session_factory is None:
return None
return DeepResearchSourceRepository(session_factory)
_default_store: DeepResearchSourceRepository | None = None
def register_default_store(store: DeepResearchSourceRepository | None) -> None:
global _default_store
_default_store = store
def get_default_store() -> DeepResearchSourceRepository | None:
return _default_store
__all__ = [
"DeepResearchSourceRepository",
"DeepResearchSourceRow",
"get_default_store",
"make_deep_research_source_store",
"register_default_store",
]
@@ -0,0 +1,68 @@
"""ORM model for deep-research source materials.
Stores every :class:`~deerflow.agents.deep_research.types.ResearchMaterial`
collected during a session, with full provenance (title, url, source,
``rec_uuid``, publish time, relevance) so report citations can trace back to
the originating DeerFlow record (§8.2 / §13.4).
Full text uses ``PortableLongText`` (MySQL ``LONGTEXT``) since web pages can be
large; the list endpoint returns summaries only (§16.5).
"""
from __future__ import annotations
from datetime import UTC, datetime
import sqlalchemy as sa
from sqlalchemy import Boolean, Float, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableLongText
class DeepResearchSourceRow(Base):
__tablename__ = "deep_research_sources"
__table_args__ = (
# MySQL requires a prefix length when indexing TEXT columns. The
# column remains unbounded so long URLs are preserved; the prefix is
# only an index implementation detail and equality filters still
# compare the complete URL value.
sa.Index("ix_deep_research_sources_url", "url", mysql_length=512),
)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
session_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
job_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
# The query that first surfaced this material.
first_seen_query: Mapped[str | None] = mapped_column(Text, nullable=True)
title: Mapped[str] = mapped_column(Text, nullable=False, default="")
url: Mapped[str | None] = mapped_column(Text, nullable=True)
raw_content: Mapped[str] = mapped_column(PortableLongText, nullable=False, default="")
snippet: Mapped[str | None] = mapped_column(Text, nullable=True)
source: Mapped[str] = mapped_column(String(256), nullable=False, default="")
source_type: Mapped[str] = mapped_column(String(32), nullable=False, default="web")
published_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
relevance_score: Mapped[float | None] = mapped_column(Float, nullable=True)
rec_uuid: Mapped[str | None] = mapped_column(String(128), nullable=True, index=True)
content_hash: Mapped[str] = mapped_column(String(64), nullable=False, default="")
# Whether this source was selected into the final context.
selected: Mapped[bool] = mapped_column(
Boolean, nullable=False, default=False, server_default=sa.false()
)
selection_reason: Mapped[str | None] = mapped_column(Text, nullable=True)
# Stable citation key assigned during report writing (e.g. "1", "2").
citation_key: Mapped[str | None] = mapped_column(String(16), nullable=True)
created_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
)
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
nullable=False,
default=lambda: datetime.now(UTC),
onupdate=lambda: datetime.now(UTC),
)
@@ -0,0 +1,243 @@
"""SQLAlchemy repository for deep-research source materials."""
from __future__ import annotations
import logging
import uuid
from typing import Any
from sqlalchemy import func, select, update
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.deep_research_sources.model import DeepResearchSourceRow
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
logger = logging.getLogger(__name__)
_JSON_COLS = ("metadata",)
class DeepResearchSourceRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
@staticmethod
def _row_to_dict(row: DeepResearchSourceRow, *, include_content: bool = True) -> dict[str, Any]:
d = row.to_dict()
if not include_content:
d.pop("raw_content", None)
d.pop("snippet", None)
return d
# ── upsert (dedupe by rec_uuid / url / content_hash) ────────────────────
async def upsert(self, *, session_id: str, job_id: str | None = None, **fields: Any) -> dict[str, Any]:
"""Insert a source, or return the existing one if rec_uuid/url/hash matches."""
# Try insert.
async with self._sf() as session:
row = DeepResearchSourceRow(session_id=session_id, job_id=job_id, **fields)
session.add(row)
try:
# The source was produced on the worker's write path. Build
# its response on that same connection so a replica-lag read
# cannot turn a successfully collected source into a failed
# research run.
await session.flush()
snapshot = self._row_to_dict(row)
await session.commit()
return snapshot
except Exception:
await session.rollback()
# Fallback: find existing by rec_uuid or url.
existing = await self._find_existing(
session_id=session_id,
rec_uuid=fields.get("rec_uuid"),
url=fields.get("url"),
content_hash=fields.get("content_hash", ""),
)
return existing or self._row_to_dict(row)
async def _find_existing(
self, *, session_id: str, rec_uuid: str | None, url: str | None, content_hash: str
) -> dict[str, Any] | None:
async with self._sf() as session:
stmt = select(DeepResearchSourceRow).where(
DeepResearchSourceRow.session_id == session_id
)
if rec_uuid:
stmt = stmt.where(DeepResearchSourceRow.rec_uuid == rec_uuid)
elif url:
stmt = stmt.where(DeepResearchSourceRow.url == url)
elif content_hash:
stmt = stmt.where(DeepResearchSourceRow.content_hash == content_hash)
else:
return None
row = (await session.execute(stmt)).scalar_one_or_none()
return self._row_to_dict(row) if row else None
# ── read ────────────────────────────────────────────────────────────────
async def get(
self,
source_id: str,
*,
user_id: str | _AutoSentinel = AUTO,
include_content: bool = True,
) -> dict[str, Any] | None:
resolved = resolve_user_id(user_id, method_name="DeepResearchSourceRepository.get")
async with self._sf() as session:
stmt = select(DeepResearchSourceRow).where(DeepResearchSourceRow.id == source_id)
if resolved is not None:
stmt = stmt.where(DeepResearchSourceRow.user_id == resolved)
row = (await session.execute(stmt)).scalar_one_or_none()
return self._row_to_dict(row, include_content=include_content) if row else None
async def list_by_session(
self,
session_id: str,
*,
user_id: str | _AutoSentinel = AUTO,
selected: bool | None = None,
limit: int = 50,
offset: int = 0,
include_content: bool = False,
) -> list[dict[str, Any]]:
resolved = resolve_user_id(user_id, method_name="DeepResearchSourceRepository.list_by_session")
async with self._sf() as session:
stmt = select(DeepResearchSourceRow).where(DeepResearchSourceRow.session_id == session_id)
if resolved is not None:
stmt = stmt.where(DeepResearchSourceRow.user_id == resolved)
if selected is not None:
stmt = stmt.where(DeepResearchSourceRow.selected == selected)
stmt = stmt.order_by(DeepResearchSourceRow.created_at.asc()).limit(limit).offset(offset)
rows = (await session.execute(stmt)).scalars().all()
return [self._row_to_dict(r, include_content=include_content) for r in rows]
async def count_by_session(
self, session_id: str, *, user_id: str | _AutoSentinel = AUTO, selected: bool | None = None
) -> int:
resolved = resolve_user_id(user_id, method_name="DeepResearchSourceRepository.count_by_session")
async with self._sf() as session:
stmt = select(func.count(DeepResearchSourceRow.id)).where(
DeepResearchSourceRow.session_id == session_id
)
if resolved is not None:
stmt = stmt.where(DeepResearchSourceRow.user_id == resolved)
if selected is not None:
stmt = stmt.where(DeepResearchSourceRow.selected == selected)
return int((await session.execute(stmt)).scalar() or 0)
async def clone_selected_to_session(
self,
source_session_id: str,
target_session_id: str,
*,
user_id: str | _AutoSentinel = AUTO,
) -> dict[str, str]:
"""Copy the selected evidence into a new report session.
Source ids are globally unique primary keys, so a report variant cannot
reuse the parent rows. Return the ``old_id -> new_id`` mapping to let
the caller repair the copied Markdown citation markers before the
variant becomes visible.
"""
resolved = resolve_user_id(
user_id,
method_name="DeepResearchSourceRepository.clone_selected_to_session",
)
async with self._sf() as session:
stmt = select(DeepResearchSourceRow).where(
DeepResearchSourceRow.session_id == source_session_id,
DeepResearchSourceRow.selected.is_(True),
)
if resolved is not None:
stmt = stmt.where(DeepResearchSourceRow.user_id == resolved)
source_rows = (await session.execute(stmt)).scalars().all()
if not source_rows:
# Collector persist marks rows selected, but an older session
# may only have unselected harvest rows. Cloning nothing would
# make 「选择结构再生成」 write with no materials.
stmt = select(DeepResearchSourceRow).where(
DeepResearchSourceRow.session_id == source_session_id,
)
if resolved is not None:
stmt = stmt.where(DeepResearchSourceRow.user_id == resolved)
source_rows = (await session.execute(stmt)).scalars().all()
id_map: dict[str, str] = {}
cloned_rows: list[DeepResearchSourceRow] = []
for source in source_rows:
cloned_id = f"drs_src_{uuid.uuid4().hex}"
id_map[source.id] = cloned_id
cloned_rows.append(
DeepResearchSourceRow(
id=cloned_id,
session_id=target_session_id,
job_id=None,
user_id=resolved if resolved is not None else source.user_id,
first_seen_query=source.first_seen_query,
title=source.title,
url=source.url,
raw_content=source.raw_content,
snippet=source.snippet,
source=source.source,
source_type=source.source_type,
published_at=source.published_at,
relevance_score=source.relevance_score,
rec_uuid=source.rec_uuid,
content_hash=source.content_hash,
selected=True,
selection_reason=source.selection_reason,
citation_key=source.citation_key,
)
)
if cloned_rows:
session.add_all(cloned_rows)
await session.flush()
await session.commit()
return id_map
# ── update selection ────────────────────────────────────────────────────
async def set_selected(
self,
source_id: str,
*,
selected: bool,
reason: str | None = None,
citation_key: str | None = None,
user_id: str | _AutoSentinel = AUTO,
) -> dict[str, Any] | None:
resolved = resolve_user_id(user_id, method_name="DeepResearchSourceRepository.set_selected")
async with self._sf() as session:
where = [DeepResearchSourceRow.id == source_id]
if resolved is not None:
where.append(DeepResearchSourceRow.user_id == resolved)
values: dict[str, Any] = {"selected": selected}
if reason is not None:
values["selection_reason"] = reason
if citation_key is not None:
values["citation_key"] = citation_key
result = await session.execute(
update(DeepResearchSourceRow).where(*where).values(**values)
)
if result.rowcount == 0:
return None
row = (
await session.execute(
select(DeepResearchSourceRow).where(DeepResearchSourceRow.id == source_id)
)
).scalar_one_or_none()
snapshot = self._row_to_dict(row) if row else None
await session.commit()
return snapshot
async def delete_by_session(self, session_id: str) -> int:
from sqlalchemy import delete
async with self._sf() as session:
result = await session.execute(
delete(DeepResearchSourceRow).where(DeepResearchSourceRow.session_id == session_id)
)
await session.commit()
return result.rowcount