feat: preserve Wiki vectors across knowledge transfers and add local BGE-M3
Unify Wiki reading, preserve metadata and graph links, add asynchronous file-backed transfers, and reuse compatible vectors. Includes regression coverage and live acceptance notes.
This commit is contained in:
1 parent
49d9ced04f
commit
60e2c72514
38 files changed
+2536
-581
No files matched your search
+1
@@ -65,6 +65,7 @@ class AssistantKnowledgeImportJobRow(Base):
|
||||
|
||||
class AssistantKnowledgeWikiPageRow(Base):
|
||||
__tablename__ = "assistant_knowledge_wiki_pages"
|
||||
metadata_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
|
||||
__table_args__ = (UniqueConstraint("base_id", "slug", name="uq_assistant_wiki_slug"),)
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
base_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
|
||||
+136
-61
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import binascii
|
||||
import difflib
|
||||
@@ -60,6 +61,12 @@ def _dict(row: Any) -> dict[str, Any]:
|
||||
data["base_kind"] = data.get("base_kind") or "custom"
|
||||
data["is_global"] = data["base_kind"] == "global"
|
||||
data["description"] = data.get("description") or ""
|
||||
if isinstance(row, AssistantKnowledgeWikiPageRow):
|
||||
metadata = data.get("metadata") or {}
|
||||
for key in ("aliases", "tags", "category_path", "parent_slug", "wiki_path", "in_links", "out_links", "source_refs", "origins"):
|
||||
data[key] = metadata.get(key, [] if key not in {"parent_slug", "wiki_path"} else None)
|
||||
data["tags"] = metadata.get("tags") or (metadata.get("page_metadata") or {}).get("tags") or []
|
||||
data["content"] = data.get("content_markdown", "")
|
||||
return data
|
||||
|
||||
|
||||
@@ -95,7 +102,11 @@ def _vector_blob_from_payload(raw: dict[str, Any]) -> tuple[bytes, int] | None:
|
||||
try:
|
||||
decoded = base64.b64decode(str(blob64), validate=True)
|
||||
vector = decode_vector(decoded, int(dimensions))
|
||||
return encode_vector(vector)
|
||||
# Our packages contain normalized float32 bytes. Preserve those
|
||||
# bytes exactly: another normalization causes round-trip drift.
|
||||
if not np.isclose(np.linalg.norm(vector), 1.0, rtol=1e-5, atol=1e-6):
|
||||
return None
|
||||
return decoded, int(dimensions)
|
||||
except (ValueError, TypeError, binascii.Error):
|
||||
return None
|
||||
value = _first_vector_value(raw)
|
||||
@@ -115,6 +126,66 @@ class AssistantKnowledgeRepository:
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def iter_export_records(self, base_id: str):
|
||||
"""Export current Wiki articles and their original vectors, without encoding."""
|
||||
base = await self.get_base(base_id)
|
||||
if not base:
|
||||
raise ValueError("助手知识库不存在")
|
||||
yield "source", {"package_version": 1, "provider": "assistant", "type": "assistant", "knowledge_base_id": base_id, "knowledge_base_name": base["name"], "export_format": "assistant-knowledge-jsonl"}
|
||||
async with self._sf() as session:
|
||||
pages = select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.base_id == base_id, AssistantKnowledgeWikiPageRow.status == "active", AssistantKnowledgeWikiPageRow.page_type != "chunk")
|
||||
stream = await session.stream_scalars(pages.execution_options(yield_per=100))
|
||||
async for page in stream:
|
||||
data = _dict(page)
|
||||
yield "wiki_page", {**(page.metadata_json or {}), **{k: data[k] for k in ("slug", "title", "page_type", "summary", "content")}, "status": "published"}
|
||||
vectors = (
|
||||
select(AssistantKnowledgeVectorRow, AssistantKnowledgeWikiPageRow.slug)
|
||||
.join(AssistantKnowledgeWikiPageRow, AssistantKnowledgeWikiPageRow.id == AssistantKnowledgeVectorRow.page_id)
|
||||
.join(AssistantKnowledgeWikiRevisionRow, AssistantKnowledgeWikiRevisionRow.id == AssistantKnowledgeWikiPageRow.current_revision_id)
|
||||
.where(
|
||||
AssistantKnowledgeVectorRow.base_id == base_id,
|
||||
AssistantKnowledgeWikiPageRow.status == "active",
|
||||
AssistantKnowledgeWikiPageRow.page_type != "chunk",
|
||||
AssistantKnowledgeVectorRow.import_job_id == AssistantKnowledgeWikiRevisionRow.import_job_id,
|
||||
)
|
||||
)
|
||||
stream = await session.stream(vectors.execution_options(yield_per=100))
|
||||
async for vector, slug in stream:
|
||||
yield (
|
||||
"vector",
|
||||
{
|
||||
"wiki_slug": slug,
|
||||
"section_index": vector.section_index,
|
||||
"heading": vector.heading,
|
||||
"section_content": vector.section_content,
|
||||
"content_hash": vector.content_hash,
|
||||
"embedding_model": vector.embedding_model,
|
||||
"embedding_fingerprint": vector.embedding_fingerprint,
|
||||
"embedding_dimensions": vector.embedding_dimensions,
|
||||
"vector_blob_base64": base64.b64encode(vector.vector_blob).decode("ascii"),
|
||||
},
|
||||
)
|
||||
stream = await session.stream(
|
||||
select(AssistantKnowledgeEntityRow, AssistantKnowledgeWikiPageRow.slug)
|
||||
.outerjoin(AssistantKnowledgeWikiPageRow, AssistantKnowledgeWikiPageRow.id == AssistantKnowledgeEntityRow.wiki_page_id)
|
||||
.where(AssistantKnowledgeEntityRow.base_id == base_id, AssistantKnowledgeEntityRow.status == "active")
|
||||
.execution_options(yield_per=100)
|
||||
)
|
||||
async for entity, slug in stream:
|
||||
yield "entity", {"id": entity.id, "name": entity.display_name, "type": entity.entity_type, "aliases": entity.aliases_json or [], "attributes": entity.attributes_json or {}, "source_wiki_slug": slug}
|
||||
stream = await session.stream(
|
||||
select(AssistantKnowledgeRelationRow, AssistantKnowledgeWikiPageRow.slug)
|
||||
.join(AssistantKnowledgeEntityRow, AssistantKnowledgeEntityRow.id == AssistantKnowledgeRelationRow.source_entity_id)
|
||||
.outerjoin(AssistantKnowledgeWikiPageRow, AssistantKnowledgeWikiPageRow.id == AssistantKnowledgeEntityRow.wiki_page_id)
|
||||
.where(AssistantKnowledgeRelationRow.base_id == base_id, AssistantKnowledgeRelationRow.status == "active")
|
||||
.execution_options(yield_per=100)
|
||||
)
|
||||
async for relation, slug in stream:
|
||||
yield (
|
||||
"relation",
|
||||
{"source_entity_id": relation.source_entity_id, "target_entity_id": relation.target_entity_id, "predicate": relation.predicate, "description": relation.description, "source_wiki_slug": slug},
|
||||
)
|
||||
|
||||
async def _base_with_stats(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
@@ -138,6 +209,7 @@ class AssistantKnowledgeRepository:
|
||||
select(func.count(AssistantKnowledgeWikiPageRow.id)).where(
|
||||
AssistantKnowledgeWikiPageRow.base_id == row.id,
|
||||
AssistantKnowledgeWikiPageRow.status == "active",
|
||||
AssistantKnowledgeWikiPageRow.page_type != "chunk",
|
||||
)
|
||||
)
|
||||
).scalar_one()
|
||||
@@ -195,14 +267,7 @@ class AssistantKnowledgeRepository:
|
||||
# Backward compatibility: earlier builds had a single active row
|
||||
# and no ``base_kind``. Promote the first surviving base into the
|
||||
# global full-library aggregation base.
|
||||
row = (
|
||||
await session.execute(
|
||||
select(AssistantKnowledgeBaseRow)
|
||||
.where(AssistantKnowledgeBaseRow.status != "deleted")
|
||||
.order_by(AssistantKnowledgeBaseRow.created_at.asc())
|
||||
.limit(1)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
row = (await session.execute(select(AssistantKnowledgeBaseRow).where(AssistantKnowledgeBaseRow.status != "deleted").order_by(AssistantKnowledgeBaseRow.created_at.asc()).limit(1))).scalar_one_or_none()
|
||||
if row is not None:
|
||||
row.base_kind = "global"
|
||||
if not row.description:
|
||||
@@ -655,7 +720,7 @@ class AssistantKnowledgeRepository:
|
||||
|
||||
entity_map: dict[str, str] = {}
|
||||
entity_rows: dict[str, tuple[AssistantKnowledgeEntityRow, dict[str, Any]]] = {}
|
||||
raw_entities = list(package.get("entities") or [])
|
||||
raw_entities = package.get("entities") or []
|
||||
for raw in raw_entities:
|
||||
display_name = str(raw.get("name") or raw.get("display_name") or "").strip()
|
||||
entity_type = str(raw.get("type") or raw.get("entity_type") or "concept")
|
||||
@@ -698,7 +763,7 @@ class AssistantKnowledgeRepository:
|
||||
)
|
||||
)
|
||||
|
||||
raw_relations = list(package.get("relations") or [])
|
||||
raw_relations = package.get("relations") or []
|
||||
relation_rows: dict[str, tuple[AssistantKnowledgeRelationRow, dict[str, Any]]] = {}
|
||||
for raw in raw_relations:
|
||||
source_entity_id = entity_map.get(str(raw.get("source_entity_id") or raw.get("source_local_id") or raw.get("source")))
|
||||
@@ -741,6 +806,10 @@ class AssistantKnowledgeRepository:
|
||||
)
|
||||
|
||||
for entity, raw in entity_rows.values():
|
||||
# Provider Wiki graph nodes already have real articles. Do not
|
||||
# turn each node into a second, empty 'entity' article.
|
||||
if raw.get("source_wiki_slug"):
|
||||
continue
|
||||
aliases = list(dict.fromkeys([*list(entity.aliases_json or []), *list(raw.get("aliases") or [])]))
|
||||
entity.aliases_json = aliases
|
||||
content = "\n".join(
|
||||
@@ -775,6 +844,8 @@ class AssistantKnowledgeRepository:
|
||||
entity.wiki_page_id = entity_page.id
|
||||
|
||||
for relation, raw in relation_rows.values():
|
||||
if raw.get("source_wiki_slug"):
|
||||
continue
|
||||
source_entity = await session.get(AssistantKnowledgeEntityRow, relation.source_entity_id)
|
||||
target_entity = await session.get(AssistantKnowledgeEntityRow, relation.target_entity_id)
|
||||
source_name_display = source_entity.display_name if source_entity else relation.source_entity_id
|
||||
@@ -796,7 +867,7 @@ class AssistantKnowledgeRepository:
|
||||
)
|
||||
relation.wiki_page_id = relation_page.id
|
||||
|
||||
pages = list(package.get("wiki_pages") or [])
|
||||
pages = package.get("wiki_pages") or []
|
||||
if not pages and package.get("wiki"):
|
||||
wiki = package["wiki"]
|
||||
page = wiki.get("skill_page") if isinstance(wiki, dict) else None
|
||||
@@ -807,7 +878,7 @@ class AssistantKnowledgeRepository:
|
||||
continue
|
||||
slug = str(raw.get("slug") or f"assistant/sources/{source.id}").strip("/")
|
||||
title = str(raw.get("title") or source_name)
|
||||
content = str(raw.get("content") or raw.get("content_markdown") or "")
|
||||
content = str(raw.get("content") or raw.get("content_md") or raw.get("content_markdown") or "")
|
||||
summary = str(raw.get("summary") or "")
|
||||
stmt = select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.base_id == base["id"], AssistantKnowledgeWikiPageRow.slug == slug)
|
||||
page = (await session.execute(stmt)).scalar_one_or_none()
|
||||
@@ -850,32 +921,35 @@ class AssistantKnowledgeRepository:
|
||||
page.current_revision_id = revision.id
|
||||
page.status = "active"
|
||||
page.updated_at = now
|
||||
metadata = {key: raw[key] for key in ("aliases", "tags", "page_metadata", "category_path", "parent_slug", "wiki_path", "in_links", "out_links", "source_refs", "chunk_refs", "version") if key in raw}
|
||||
source_meta = package.get("source") or {}
|
||||
origin_mapping = mapping_id
|
||||
if not origin_mapping and (source_meta.get("provider") or source_meta.get("type")) == "weknora":
|
||||
origin_mapping = source_meta.get("knowledge_base_id")
|
||||
origins = [*((page.metadata_json or {}).get("origins") or []), *(raw.get("origins") or [])]
|
||||
origin = {"mapping_id": origin_mapping, "source_key": source_key, "source_name": source_name, "slug": slug}
|
||||
origins = [value for value in origins if isinstance(value, dict) and value.get("source_key") != source_key]
|
||||
metadata["origins"] = list({(v.get("mapping_id"), v.get("source_key"), v.get("slug")): v for v in [*origins, origin]}.values())
|
||||
page.metadata_json = metadata
|
||||
page_ids_by_slug[slug] = page.id
|
||||
for entity, raw_entity in entity_rows.values():
|
||||
if raw_entity.get("source_wiki_slug") == slug:
|
||||
entity.wiki_page_id = page.id
|
||||
if len(page_ids_by_slug) % 50 == 0:
|
||||
await session.flush()
|
||||
|
||||
chunks = list(package.get("chunks") or [])
|
||||
chunks = package.get("chunks") or []
|
||||
chunk_pages = 0
|
||||
imported_vectors = 0
|
||||
for index, raw in enumerate(chunks):
|
||||
if not isinstance(raw, dict):
|
||||
continue
|
||||
content = str(
|
||||
raw.get("content")
|
||||
or raw.get("section_content")
|
||||
or raw.get("text")
|
||||
or raw.get("chunk_content")
|
||||
or ""
|
||||
).strip()
|
||||
content = str(raw.get("content") or raw.get("section_content") or raw.get("text") or raw.get("chunk_content") or "").strip()
|
||||
if not content:
|
||||
continue
|
||||
raw_chunk_id = str(raw.get("id") or raw.get("chunk_id") or raw.get("remote_id") or f"{source.id}:{index}")
|
||||
document_id = str(raw.get("source_document_id") or raw.get("knowledge_id") or raw.get("document_id") or "")
|
||||
title = str(
|
||||
raw.get("title")
|
||||
or raw.get("knowledge_title")
|
||||
or raw.get("filename")
|
||||
or raw.get("knowledge_filename")
|
||||
or f"{source_name} 片段 {index + 1}"
|
||||
)
|
||||
title = str(raw.get("title") or raw.get("knowledge_title") or raw.get("filename") or raw.get("knowledge_filename") or f"{source_name} 片段 {index + 1}")
|
||||
heading = str(raw.get("heading") or raw.get("section_title") or "")
|
||||
summary = content[:240]
|
||||
page_content = "\n".join(
|
||||
@@ -919,7 +993,7 @@ class AssistantKnowledgeRepository:
|
||||
import_job_id=job.id,
|
||||
source_vector_id=str(raw.get("vector_id") or raw.get("embedding_id") or raw_chunk_id),
|
||||
source_chunk_id=raw_chunk_id,
|
||||
section_index=int(raw.get("section_index") or raw.get("chunk_index") or index),
|
||||
section_index=int(next((raw[key] for key in ("section_index", "chunk_index") if raw.get(key) is not None), index)),
|
||||
heading=heading or None,
|
||||
section_content=content,
|
||||
content_hash=str(raw.get("content_hash") or _content_hash(content)),
|
||||
@@ -933,19 +1007,19 @@ class AssistantKnowledgeRepository:
|
||||
imported_vectors += 1
|
||||
chunk_pages += 1
|
||||
|
||||
for index, raw in enumerate(list(package.get("vectors") or [])):
|
||||
for index, raw in enumerate(package.get("vectors") or []):
|
||||
if not isinstance(raw, dict):
|
||||
continue
|
||||
raise ValueError("向量记录必须为对象")
|
||||
slug = str(raw.get("wiki_slug") or raw.get("page_slug") or raw.get("slug") or "").strip("/")
|
||||
page_id = page_ids_by_slug.get(slug)
|
||||
if not page_id:
|
||||
continue
|
||||
raise ValueError(f"向量引用不存在的 Wiki:{slug}")
|
||||
content = str(raw.get("section_content") or raw.get("content") or raw.get("text") or "").strip()
|
||||
if not content:
|
||||
continue
|
||||
vector_blob = _vector_blob_from_payload(raw)
|
||||
if vector_blob is None:
|
||||
continue
|
||||
raise ValueError(f"Wiki 向量格式无效:{slug}")
|
||||
blob, dimensions = vector_blob
|
||||
raw_chunk_id = str(raw.get("source_chunk_id") or raw.get("chunk_id") or "")
|
||||
session.add(
|
||||
@@ -957,7 +1031,7 @@ class AssistantKnowledgeRepository:
|
||||
import_job_id=job.id,
|
||||
source_vector_id=str(raw.get("id") or raw.get("vector_id") or f"{slug}:{index}"),
|
||||
source_chunk_id=raw_chunk_id or None,
|
||||
section_index=int(raw.get("section_index") or index),
|
||||
section_index=int(raw["section_index"]) if raw.get("section_index") is not None else index,
|
||||
heading=str(raw.get("heading") or "") or None,
|
||||
section_content=content,
|
||||
content_hash=str(raw.get("content_hash") or _content_hash(content)),
|
||||
@@ -969,9 +1043,11 @@ class AssistantKnowledgeRepository:
|
||||
)
|
||||
)
|
||||
imported_vectors += 1
|
||||
if imported_vectors % 50 == 0:
|
||||
await session.flush()
|
||||
|
||||
counts = {
|
||||
"wiki_pages": len(pages) + len(entity_rows) + len(relation_rows),
|
||||
"wiki_pages": len(pages) + sum(not raw.get("source_wiki_slug") for _, raw in entity_rows.values()) + sum(not raw.get("source_wiki_slug") for _, raw in relation_rows.values()),
|
||||
"documents": len(package.get("documents") or []),
|
||||
"chunks": len(chunks),
|
||||
"chunk_pages": chunk_pages,
|
||||
@@ -1008,6 +1084,7 @@ class AssistantKnowledgeRepository:
|
||||
query: str | None = None,
|
||||
page_type: str | None = None,
|
||||
limit: int = 100,
|
||||
offset: int = 0,
|
||||
include_chunks: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
base = await self.get_base(base_id)
|
||||
@@ -1024,7 +1101,7 @@ class AssistantKnowledgeRepository:
|
||||
stmt = stmt.where(or_(AssistantKnowledgeWikiPageRow.title.ilike(needle), AssistantKnowledgeWikiPageRow.search_text.ilike(needle)))
|
||||
if page_type:
|
||||
stmt = stmt.where(AssistantKnowledgeWikiPageRow.page_type == page_type)
|
||||
stmt = stmt.order_by(AssistantKnowledgeWikiPageRow.updated_at.desc()).limit(limit)
|
||||
stmt = stmt.order_by(AssistantKnowledgeWikiPageRow.slug, AssistantKnowledgeWikiPageRow.id).offset(offset).limit(limit)
|
||||
async with self._sf() as session:
|
||||
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
|
||||
|
||||
@@ -1070,10 +1147,7 @@ class AssistantKnowledgeRepository:
|
||||
or_(
|
||||
AssistantKnowledgeWikiPageRow.title.ilike(needle),
|
||||
AssistantKnowledgeWikiPageRow.search_text.ilike(needle),
|
||||
*[
|
||||
AssistantKnowledgeWikiPageRow.search_text.ilike(f"%{token}%")
|
||||
for token in tokens[:8]
|
||||
],
|
||||
*[AssistantKnowledgeWikiPageRow.search_text.ilike(f"%{token}%") for token in tokens[:8]],
|
||||
)
|
||||
)
|
||||
stmt = stmt.order_by(AssistantKnowledgeWikiPageRow.updated_at.desc()).limit(max(limit * 8, limit))
|
||||
@@ -1098,6 +1172,7 @@ class AssistantKnowledgeRepository:
|
||||
results = rows[:limit]
|
||||
for row in results:
|
||||
row["score"] = score(row)
|
||||
row["retrieval_mode"] = "keyword"
|
||||
return results
|
||||
|
||||
async def _search_pages_by_vector(
|
||||
@@ -1119,31 +1194,36 @@ class AssistantKnowledgeRepository:
|
||||
AssistantKnowledgeWikiPageRow,
|
||||
AssistantKnowledgeWikiPageRow.id == AssistantKnowledgeVectorRow.page_id,
|
||||
)
|
||||
.join(AssistantKnowledgeWikiRevisionRow, AssistantKnowledgeWikiRevisionRow.id == AssistantKnowledgeWikiPageRow.current_revision_id)
|
||||
.where(
|
||||
AssistantKnowledgeVectorRow.base_id.in_(base_ids),
|
||||
AssistantKnowledgeVectorRow.import_job_id == AssistantKnowledgeWikiRevisionRow.import_job_id,
|
||||
AssistantKnowledgeVectorRow.embedding_dimensions == dimensions,
|
||||
AssistantKnowledgeVectorRow.embedding_fingerprint == embedding_client.fingerprint,
|
||||
AssistantKnowledgeWikiPageRow.status == "active",
|
||||
)
|
||||
.order_by(AssistantKnowledgeVectorRow.created_at.desc())
|
||||
)
|
||||
if not include_chunks:
|
||||
stmt = stmt.where(AssistantKnowledgeWikiPageRow.page_type != "chunk")
|
||||
async with self._sf() as session:
|
||||
rows = list((await session.execute(stmt)).all())
|
||||
if not rows:
|
||||
return []
|
||||
tokens = [token for token in query.strip().lower().split() if token]
|
||||
ranked: list[tuple[float, AssistantKnowledgeVectorRow, AssistantKnowledgeWikiPageRow]] = []
|
||||
for vector, page in rows:
|
||||
try:
|
||||
stored = decode_vector(vector.vector_blob, int(vector.embedding_dimensions))
|
||||
except ValueError:
|
||||
continue
|
||||
score = float(np.dot(query_vector, stored))
|
||||
if tokens:
|
||||
haystack = f"{page.title}\n{page.summary}\n{vector.heading or ''}\n{vector.section_content}".lower()
|
||||
score += min(sum(1 for token in tokens if token in haystack), 4) * 0.02
|
||||
ranked.append((score, vector, page))
|
||||
# Bounded scan: keep only the best section per page, and at most limit
|
||||
# candidates. Never load a multi-GB vector library into one matrix.
|
||||
async with self._sf() as session:
|
||||
stream = await session.stream(stmt.execution_options(yield_per=256))
|
||||
async for batch in stream.partitions(256):
|
||||
|
||||
def score_batch():
|
||||
values = []
|
||||
for vector, page in batch:
|
||||
values.append((float(np.dot(query_vector, decode_vector(vector.vector_blob, dimensions))), vector, page))
|
||||
return values
|
||||
|
||||
candidates = ranked + await asyncio.to_thread(score_batch)
|
||||
best = {}
|
||||
for item in sorted(candidates, key=lambda item: item[0], reverse=True):
|
||||
best.setdefault(item[2].id, item)
|
||||
ranked = list(best.values())[:limit]
|
||||
ranked.sort(key=lambda item: item[0], reverse=True)
|
||||
|
||||
results: list[dict[str, Any]] = []
|
||||
@@ -1320,12 +1400,7 @@ class AssistantKnowledgeRepository:
|
||||
if base is None:
|
||||
return []
|
||||
async with self._sf() as session:
|
||||
stmt = (
|
||||
select(AssistantKnowledgeImportJobRow)
|
||||
.where(AssistantKnowledgeImportJobRow.base_id == base["id"])
|
||||
.order_by(AssistantKnowledgeImportJobRow.created_at.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
stmt = select(AssistantKnowledgeImportJobRow).where(AssistantKnowledgeImportJobRow.base_id == base["id"]).order_by(AssistantKnowledgeImportJobRow.created_at.desc()).limit(limit)
|
||||
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
|
||||
|
||||
async def get_job(self, job_id: str) -> dict[str, Any] | None:
|
||||
|
||||
+1
-1
@@ -24,7 +24,7 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
|
||||
async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None:
|
||||
row = next((p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and p["slug_hash"] == slug_hash), None)
|
||||
return dict(row) if row else None
|
||||
return {**row, "vector_count": len(self.vectors.get(row["id"], []))} if row else None
|
||||
|
||||
async def get_page(self, mapping_id: str, slug: str) -> dict[str, Any] | None:
|
||||
row = next((p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and p["slug"] == slug), None)
|
||||
|
||||
+4
-3
@@ -41,10 +41,11 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
return row
|
||||
|
||||
async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None:
|
||||
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == slug_hash)
|
||||
count = select(func.count(LlmWikiWikiVectorRow.id)).where(LlmWikiWikiVectorRow.page_id == LlmWikiWikiPageRow.id).correlate(LlmWikiWikiPageRow).scalar_subquery()
|
||||
stmt = select(LlmWikiWikiPageRow, count).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == slug_hash)
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
return _dict(row) if row else None
|
||||
result = (await session.execute(stmt)).one_or_none()
|
||||
return {**_dict(result[0]), "vector_count": result[1]} if result else None
|
||||
|
||||
async def get_page(self, mapping_id: str, slug: str) -> dict[str, Any] | None:
|
||||
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug == slug)
|
||||
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
"""Preserve source Wiki metadata during package round trips."""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from deerflow.persistence.types import PortableJSON
|
||||
|
||||
revision = "20260903_01"
|
||||
down_revision = ("20260902_01", "20260831_01")
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
columns = {col["name"] for col in sa.inspect(op.get_bind()).get_columns("assistant_knowledge_wiki_pages")}
|
||||
if "metadata_json" not in columns:
|
||||
op.add_column("assistant_knowledge_wiki_pages", sa.Column("metadata_json", PortableJSON(), nullable=True))
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_column("assistant_knowledge_wiki_pages", "metadata_json")
|
||||
Reference in new issue
Block a user