Merge branch 'master' of http://192.168.0.200/knowledge-base-bj/deerflow-code into master-nj
This commit is contained in:
commit
5c0864bf0a
40 files changed
+1200
-179
No files matched your search
+1
@@ -18,6 +18,7 @@ class LlmCallMetricRow(Base):
|
||||
__table_args__ = (
|
||||
Index("ix_llm_call_metrics_created_user", "created_at", "user_id"),
|
||||
Index("ix_llm_call_metrics_run_created", "run_id", "created_at"),
|
||||
Index("ix_llm_call_metrics_created_model_user", "created_at", "model_name", "user_id"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
|
||||
+6
-2
@@ -6,8 +6,12 @@ from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWiki
|
||||
from deerflow.persistence.llmwiki_index.sql import SqlLlmWikiIndexStore
|
||||
|
||||
|
||||
def make_llmwiki_index_store(session_factory):
|
||||
return MemoryLlmWikiIndexStore() if session_factory is None else SqlLlmWikiIndexStore(session_factory)
|
||||
def make_llmwiki_index_store(session_factory, *, write_batch_size: int = 50):
|
||||
return (
|
||||
MemoryLlmWikiIndexStore()
|
||||
if session_factory is None
|
||||
else SqlLlmWikiIndexStore(session_factory, write_batch_size=write_batch_size)
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
|
||||
+13
@@ -44,11 +44,24 @@ class LlmWikiIndexStore(ABC):
|
||||
@abstractmethod
|
||||
async def renew_sync_lease(self, mapping_id: str, *, owner: str, lease_seconds: int) -> bool: ...
|
||||
|
||||
@abstractmethod
|
||||
async def update_sync_progress(
|
||||
self,
|
||||
mapping_id: str,
|
||||
*,
|
||||
owner: str,
|
||||
sync_id: str,
|
||||
remote_page_count: int,
|
||||
) -> bool: ...
|
||||
|
||||
@abstractmethod
|
||||
async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]: ...
|
||||
|
||||
@abstractmethod
|
||||
async def list_page_index_status(self, mapping_id: str) -> list[dict[str, Any]]: ...
|
||||
|
||||
@abstractmethod
|
||||
async def mark_rebuild_required(self, mapping_ids: list[str], fingerprint: str) -> int: ...
|
||||
+42
@@ -123,6 +123,21 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
state["lease_expires_at"] = _now() + timedelta(seconds=lease_seconds)
|
||||
return True
|
||||
|
||||
async def update_sync_progress(
|
||||
self,
|
||||
mapping_id: str,
|
||||
*,
|
||||
owner: str,
|
||||
sync_id: str,
|
||||
remote_page_count: int,
|
||||
) -> bool:
|
||||
state = self._state(mapping_id)
|
||||
if state.get("lease_owner") != owner or state.get("active_sync_id") != sync_id:
|
||||
return False
|
||||
state["remote_page_count"] = max(0, remote_page_count)
|
||||
state["updated_at"] = _now()
|
||||
return True
|
||||
|
||||
async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None:
|
||||
state = self._state(mapping_id)
|
||||
if state.get("lease_owner") != owner or state.get("active_sync_id") != sync_id:
|
||||
@@ -150,6 +165,33 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
wanted = set(mapping_ids or [])
|
||||
return [dict(value) for key, value in self.states.items() if not wanted or key in wanted]
|
||||
|
||||
async def list_page_index_status(self, mapping_id: str) -> list[dict[str, Any]]:
|
||||
result = []
|
||||
for page in sorted(
|
||||
(
|
||||
page
|
||||
for page in self.pages.values()
|
||||
if page["knowledge_base_mapping_id"] == mapping_id and not page.get("is_remote_deleted")
|
||||
),
|
||||
key=lambda page: (str(page.get("wiki_path") or ""), str(page.get("slug") or "")),
|
||||
):
|
||||
result.append(
|
||||
{
|
||||
"id": page["id"],
|
||||
"slug": page["slug"],
|
||||
"title": page["title"],
|
||||
"page_type": page["page_type"],
|
||||
"status": page["status"],
|
||||
"index_status": page["index_status"],
|
||||
"vector_count": len(self.vectors.get(page["id"], [])),
|
||||
"indexed_at": page.get("indexed_at"),
|
||||
"last_error": page.get("last_error"),
|
||||
"last_seen_sync_id": page.get("last_seen_sync_id"),
|
||||
"updated_at": page.get("updated_at"),
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
async def mark_rebuild_required(self, mapping_ids: list[str], fingerprint: str) -> int:
|
||||
for mapping_id in mapping_ids:
|
||||
state = self._state(mapping_id)
|
||||
|
||||
+88
-20
@@ -6,7 +6,7 @@ from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import case, delete, func, or_, select, update
|
||||
from sqlalchemy import case, delete, func, insert, or_, select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
@@ -27,8 +27,9 @@ def _dict(row) -> dict[str, Any]:
|
||||
|
||||
|
||||
class SqlLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession], *, write_batch_size: int = 50) -> None:
|
||||
self._sf = session_factory
|
||||
self.write_batch_size = max(1, min(int(write_batch_size), 500))
|
||||
|
||||
async def _ensure_state(self, session: AsyncSession, mapping_id: str) -> LlmWikiWikiSyncStateRow:
|
||||
row = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
|
||||
@@ -82,24 +83,29 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
row.updated_at = now
|
||||
await session.flush()
|
||||
await session.execute(delete(LlmWikiWikiVectorRow).where(LlmWikiWikiVectorRow.page_id == row.id))
|
||||
session.add_all(
|
||||
[
|
||||
LlmWikiWikiVectorRow(
|
||||
id=str(uuid4()),
|
||||
page_id=row.id,
|
||||
knowledge_base_mapping_id=mapping_id,
|
||||
section_index=int(item["section_index"]),
|
||||
heading=item.get("heading"),
|
||||
section_content=str(item["section_content"]),
|
||||
content_hash=str(item["content_hash"]),
|
||||
embedding_fingerprint=fingerprint,
|
||||
embedding_dimensions=int(item["embedding_dimensions"]),
|
||||
vector_blob=bytes(item["vector_blob"]),
|
||||
created_at=now,
|
||||
)
|
||||
for item in vectors
|
||||
]
|
||||
)
|
||||
vector_rows = [
|
||||
{
|
||||
"id": str(uuid4()),
|
||||
"page_id": row.id,
|
||||
"knowledge_base_mapping_id": mapping_id,
|
||||
"section_index": int(item["section_index"]),
|
||||
"heading": item.get("heading"),
|
||||
"section_content": str(item["section_content"]),
|
||||
"content_hash": str(item["content_hash"]),
|
||||
"embedding_fingerprint": fingerprint,
|
||||
"embedding_dimensions": int(item["embedding_dimensions"]),
|
||||
"vector_blob": bytes(item["vector_blob"]),
|
||||
"created_at": now,
|
||||
}
|
||||
for item in vectors
|
||||
]
|
||||
# Keep the page replacement atomic, but cap each SQL statement so
|
||||
# large Wiki pages cannot flood the database with one huge insert.
|
||||
for start in range(0, len(vector_rows), self.write_batch_size):
|
||||
await session.execute(
|
||||
insert(LlmWikiWikiVectorRow),
|
||||
vector_rows[start : start + self.write_batch_size],
|
||||
)
|
||||
state = await self._ensure_state(session, mapping_id)
|
||||
state.index_revision = int(state.index_revision or 0) + 1
|
||||
state.embedding_fingerprint = fingerprint
|
||||
@@ -229,6 +235,27 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
await session.commit()
|
||||
return bool(result.rowcount)
|
||||
|
||||
async def update_sync_progress(
|
||||
self,
|
||||
mapping_id: str,
|
||||
*,
|
||||
owner: str,
|
||||
sync_id: str,
|
||||
remote_page_count: int,
|
||||
) -> bool:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(LlmWikiWikiSyncStateRow)
|
||||
.where(
|
||||
LlmWikiWikiSyncStateRow.knowledge_base_mapping_id == mapping_id,
|
||||
LlmWikiWikiSyncStateRow.lease_owner == owner,
|
||||
LlmWikiWikiSyncStateRow.active_sync_id == sync_id,
|
||||
)
|
||||
.values(remote_page_count=max(0, remote_page_count), updated_at=_now())
|
||||
)
|
||||
await session.commit()
|
||||
return bool(result.rowcount)
|
||||
|
||||
async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None:
|
||||
async with self._sf() as session:
|
||||
state = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
|
||||
@@ -280,6 +307,47 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
async with self._sf() as session:
|
||||
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
|
||||
|
||||
async def list_page_index_status(self, mapping_id: str) -> list[dict[str, Any]]:
|
||||
vector_counts = (
|
||||
select(
|
||||
LlmWikiWikiVectorRow.page_id.label("page_id"),
|
||||
func.count(LlmWikiWikiVectorRow.id).label("vector_count"),
|
||||
)
|
||||
.where(LlmWikiWikiVectorRow.knowledge_base_mapping_id == mapping_id)
|
||||
.group_by(LlmWikiWikiVectorRow.page_id)
|
||||
.subquery()
|
||||
)
|
||||
stmt = (
|
||||
select(LlmWikiWikiPageRow, func.coalesce(vector_counts.c.vector_count, 0))
|
||||
.outerjoin(vector_counts, vector_counts.c.page_id == LlmWikiWikiPageRow.id)
|
||||
.where(
|
||||
LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id,
|
||||
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
|
||||
)
|
||||
.order_by(LlmWikiWikiPageRow.wiki_path, LlmWikiWikiPageRow.slug)
|
||||
)
|
||||
async with self._sf() as session:
|
||||
rows = (await session.execute(stmt)).all()
|
||||
result: list[dict[str, Any]] = []
|
||||
for row, vector_count in rows:
|
||||
page = _dict(row)
|
||||
result.append(
|
||||
{
|
||||
"id": page["id"],
|
||||
"slug": page["slug"],
|
||||
"title": page["title"],
|
||||
"page_type": page["page_type"],
|
||||
"status": page["status"],
|
||||
"index_status": page["index_status"],
|
||||
"vector_count": int(vector_count or 0),
|
||||
"indexed_at": page.get("indexed_at"),
|
||||
"last_error": page.get("last_error"),
|
||||
"last_seen_sync_id": page.get("last_seen_sync_id"),
|
||||
"updated_at": page.get("updated_at"),
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
async def mark_rebuild_required(self, mapping_ids: list[str], fingerprint: str) -> int:
|
||||
if not mapping_ids:
|
||||
return 0
|
||||
|
||||
+4
-4
@@ -1,7 +1,7 @@
|
||||
"""Create administrator-managed exact-match fixed questions.
|
||||
|
||||
Revision ID: 20260821_01
|
||||
Revises: 20260816_01
|
||||
Revision ID: 20260821_03
|
||||
Revises: 20260821_01
|
||||
Create Date: 2026-08-21
|
||||
"""
|
||||
|
||||
@@ -11,8 +11,8 @@ import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import mysql
|
||||
|
||||
revision = "20260821_01"
|
||||
down_revision = "20260816_01"
|
||||
revision = "20260821_03"
|
||||
down_revision = "20260821_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
+4
-4
@@ -1,7 +1,7 @@
|
||||
"""Add configurable fixed-answer streaming speed.
|
||||
|
||||
Revision ID: 20260821_02
|
||||
Revises: 20260821_01
|
||||
Revision ID: 20260821_04
|
||||
Revises: 20260821_03
|
||||
Create Date: 2026-08-21
|
||||
"""
|
||||
|
||||
@@ -10,8 +10,8 @@ from __future__ import annotations
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "20260821_02"
|
||||
down_revision = "20260821_01"
|
||||
revision = "20260821_04"
|
||||
down_revision = "20260821_03"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
+2
-2
@@ -1,7 +1,7 @@
|
||||
"""Add DeerFlow-local WeKnora Wiki mirror and vector index.
|
||||
|
||||
Revision ID: 20260823_01
|
||||
Revises: 20260821_02
|
||||
Revises: 20260821_04
|
||||
Create Date: 2026-08-23
|
||||
"""
|
||||
|
||||
@@ -12,7 +12,7 @@ from alembic import op
|
||||
from sqlalchemy.dialects import mysql
|
||||
|
||||
revision = "20260823_01"
|
||||
down_revision = "20260821_02"
|
||||
down_revision = "20260821_04"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
"""Merge report-structure and local-Wiki heads, repairing the old duplicate stamp.
|
||||
|
||||
Revision ID: 20260825_01
|
||||
Revises: 20260822_02, 20260823_01
|
||||
Create Date: 2026-08-25
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "20260825_01"
|
||||
down_revision = ("20260822_02", "20260823_01")
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# An older build accidentally used 20260821_01 for two migrations. A DB
|
||||
# stamped by the fixed-question variant may therefore still lack this
|
||||
# report column even though Alembic considers the surviving 01 applied.
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
if inspector.has_table("report_structures"):
|
||||
columns = {column["name"] for column in inspector.get_columns("report_structures")}
|
||||
if "structure_mode" not in columns:
|
||||
op.add_column(
|
||||
"report_structures",
|
||||
sa.Column("structure_mode", sa.String(length=16), nullable=False, server_default="adaptive"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
pass
|
||||
@@ -52,4 +52,6 @@ class RunRow(Base):
|
||||
Index("ix_runs_created_user_status", "created_at", "user_id", "status"),
|
||||
Index("ix_runs_user_created", "user_id", "created_at"),
|
||||
Index("ix_runs_thread_created", "thread_id", "created_at"),
|
||||
Index("ix_runs_created_assistant_user", "created_at", "assistant_id", "user_id"),
|
||||
Index("ix_runs_created_model_user", "created_at", "model_name", "user_id"),
|
||||
)
|
||||
+2
@@ -24,6 +24,8 @@ class ToolCallMetricRow(Base):
|
||||
__table_args__ = (
|
||||
Index("ix_tool_call_metrics_created_user", "created_at", "user_id"),
|
||||
Index("ix_tool_call_metrics_run_created", "run_id", "created_at"),
|
||||
Index("ix_tool_call_metrics_created_skill_user", "created_at", "skill_name", "user_id"),
|
||||
Index("ix_tool_call_metrics_created_tool_user", "created_at", "tool_name", "user_id"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
|
||||
Reference in new issue
Block a user