feat: add compact toolbox and public wiki vector search
This commit is contained in:
1 parent
9f29ecae9e
commit
d35d075669
29 files changed
+757
-128
No files matched your search
@@ -17,8 +17,8 @@ from pydantic import BaseModel, Field
|
||||
|
||||
from app.gateway.deps import get_current_user, get_optional_user_from_request
|
||||
from deerflow.persistence.compact_menu_overrides import (
|
||||
CompactMenuOverrideStore,
|
||||
DEFAULT_COMPACT_MENU_TYPE,
|
||||
CompactMenuOverrideStore,
|
||||
normalize_compact_menu_type,
|
||||
)
|
||||
|
||||
@@ -27,7 +27,7 @@ router = APIRouter(prefix="/api/compact-menu-overrides", tags=["compact-menu-ove
|
||||
|
||||
class CompactMenuOverride(BaseModel):
|
||||
node_id: str = Field(..., min_length=1, max_length=128)
|
||||
placement: Literal["sidebar", "settings"] = "sidebar"
|
||||
placement: Literal["sidebar", "settings", "toolbox"] = "sidebar"
|
||||
sort_order: int = 0
|
||||
label: str | None = Field(default=None, max_length=255)
|
||||
|
||||
|
||||
@@ -1151,35 +1151,15 @@ html.deerflow-source-detail-open [data-deerflow-source-detail-panel="true"] [cla
|
||||
if (!proxyUrl) return;
|
||||
if (
|
||||
image.dataset.deerflowProtectedSource === protectedSrc &&
|
||||
String(image.getAttribute("src") || "").startsWith("blob:")
|
||||
String(image.getAttribute("src") || "") === proxyUrl
|
||||
) return;
|
||||
const attempts = Number(image.dataset.deerflowProtectedAttempts || "0");
|
||||
if (attempts >= 3 || image.dataset.deerflowProtectedLoading === "1") return;
|
||||
image.dataset.deerflowProtectedLoading = "1";
|
||||
image.dataset.deerflowProtectedAttempts = String(attempts + 1);
|
||||
nativeFetch(proxyUrl, {{ credentials: "same-origin", headers: {{ Accept: "image/*" }} }})
|
||||
.then((response) => {{
|
||||
if (!response.ok) throw new Error(`protected image request failed (${{response.status}})`);
|
||||
return response.blob();
|
||||
}})
|
||||
.then((blob) => {{
|
||||
const objectUrl = URL.createObjectURL(blob);
|
||||
image.dataset.deerflowProtectedSource = protectedSrc;
|
||||
image.removeAttribute("data-img-loading");
|
||||
image.removeAttribute("data-wiki-image-error");
|
||||
image.src = objectUrl;
|
||||
image.addEventListener("load", () => URL.revokeObjectURL(objectUrl), {{ once: true }});
|
||||
}})
|
||||
.catch(() => {{
|
||||
image.dataset.wikiImageError = "true";
|
||||
window.setTimeout(() => {{
|
||||
delete image.dataset.deerflowProtectedLoading;
|
||||
hydrateProtectedImage(image);
|
||||
}}, 1200 * (attempts + 1));
|
||||
}})
|
||||
.finally(() => {{
|
||||
delete image.dataset.deerflowProtectedLoading;
|
||||
}});
|
||||
// Keep the real HTTP proxy URL on the DOM node. WeKnora's preview/lightbox
|
||||
// reuses the thumbnail's src when the user clicks it; a short-lived blob URL
|
||||
// may render once and then become invalid before the preview opens.
|
||||
image.dataset.deerflowProtectedSource = protectedSrc;
|
||||
image.removeAttribute("data-img-loading");
|
||||
image.removeAttribute("data-wiki-image-error");
|
||||
image.src = proxyUrl;
|
||||
}}
|
||||
function hydrateProtectedImages(scope = document) {{
|
||||
if (scope?.matches?.("img[data-protected-src]")) hydrateProtectedImage(scope);
|
||||
@@ -1197,7 +1177,7 @@ html.deerflow-source-detail-open [data-deerflow-source-detail-panel="true"] [cla
|
||||
childList: true,
|
||||
subtree: true,
|
||||
attributes: true,
|
||||
attributeFilter: ["data-protected-src"]
|
||||
attributeFilter: ["data-protected-src", "src"]
|
||||
}});
|
||||
hydrateProtectedImages();
|
||||
|
||||
|
||||
+152
@@ -0,0 +1,152 @@
|
||||
"""Anonymous vector search over published, locally indexed WeKnora Wiki data."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from urllib.parse import quote, urlencode
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from deerflow.integrations.weknora.local_index.search import WikiIndexError
|
||||
|
||||
router = APIRouter(tags=["public-knowledge-vector-search"])
|
||||
|
||||
|
||||
class PublicKnowledgeVectorSearchRequest(BaseModel):
|
||||
query: str = Field(..., min_length=1, max_length=10_000)
|
||||
top_k: int = Field(default=10, ge=1, le=100)
|
||||
include_page_content: bool = False
|
||||
|
||||
|
||||
def _is_conversation_deposit(mapping: dict[str, Any]) -> bool:
|
||||
return str(mapping.get("owner_user_id") or "") == "system" and str(mapping.get("name") or "").strip() == "对话沉淀"
|
||||
|
||||
|
||||
async def _published_mappings(request: Request) -> list[dict[str, Any]]:
|
||||
store = getattr(request.app.state, "llmwiki_store", None)
|
||||
if store is None:
|
||||
raise HTTPException(status_code=503, detail={"code": "LLMWIKI_STORE_UNAVAILABLE"})
|
||||
rows = await store.list_visible("system", scope="public", is_admin=False)
|
||||
return [row for row in rows if row.get("publication_status") == "published" and not _is_conversation_deposit(row)]
|
||||
|
||||
|
||||
def _unique_strings(values: list[str]) -> list[str]:
|
||||
return list(dict.fromkeys(value for value in values if value))
|
||||
|
||||
|
||||
def _reference_metadata(item: dict[str, Any]) -> dict[str, Any]:
|
||||
source_refs = item.get("source_refs") if isinstance(item.get("source_refs"), list) else []
|
||||
chunk_refs = item.get("chunk_refs") if isinstance(item.get("chunk_refs"), list) else []
|
||||
document_ids: list[str] = []
|
||||
document_names: list[str] = []
|
||||
chunk_ids: list[str] = []
|
||||
source_ref_ids: list[str] = []
|
||||
|
||||
for ref in source_refs:
|
||||
if isinstance(ref, dict):
|
||||
document_id = str(ref.get("document_id") or ref.get("knowledge_id") or ref.get("documentId") or ref.get("knowledgeId") or "").strip()
|
||||
document_name = str(
|
||||
ref.get("document_name") or ref.get("knowledge_title") or ref.get("documentName") or ref.get("knowledgeTitle") or ref.get("file_name") or ref.get("filename") or ref.get("title") or ref.get("name") or ""
|
||||
).strip()
|
||||
chunk_id = str(ref.get("chunk_id") or ref.get("chunkId") or "").strip()
|
||||
if document_id:
|
||||
document_ids.append(document_id)
|
||||
if document_name:
|
||||
document_names.append(document_name)
|
||||
if chunk_id:
|
||||
chunk_ids.append(chunk_id)
|
||||
elif isinstance(ref, (str, int)) and str(ref).strip():
|
||||
source_ref_ids.append(str(ref).strip())
|
||||
|
||||
for ref in chunk_refs:
|
||||
if isinstance(ref, dict):
|
||||
chunk_id = str(ref.get("chunk_id") or ref.get("chunkId") or ref.get("id") or "").strip()
|
||||
else:
|
||||
chunk_id = str(ref or "").strip()
|
||||
if chunk_id:
|
||||
chunk_ids.append(chunk_id)
|
||||
|
||||
document_ids = _unique_strings(document_ids)
|
||||
document_names = _unique_strings(document_names)
|
||||
chunk_ids = _unique_strings(chunk_ids)
|
||||
source_ref_ids = _unique_strings(source_ref_ids)
|
||||
return {
|
||||
"document_id": document_ids[0] if document_ids else None,
|
||||
"document_name": document_names[0] if document_names else None,
|
||||
"document_ids": document_ids,
|
||||
"document_names": document_names,
|
||||
"chunk_id": chunk_ids[0] if chunk_ids else None,
|
||||
"chunk_ids": chunk_ids,
|
||||
"source_ref_ids": source_ref_ids,
|
||||
}
|
||||
|
||||
|
||||
def _wiki_link(frontend_base_url: str, mapping_id: str, wiki_slug: str) -> tuple[str, str]:
|
||||
query = urlencode({"wikiSlug": wiki_slug})
|
||||
route = f"/#/page/workspace/knowledge/{quote(mapping_id, safe='')}/wiki/view?{query}"
|
||||
return route, f"{frontend_base_url.rstrip('/')}{route}"
|
||||
|
||||
|
||||
@router.post("/api/knowledge/vector-search")
|
||||
async def public_knowledge_vector_search(
|
||||
request: Request,
|
||||
body: PublicKnowledgeVectorSearchRequest,
|
||||
) -> dict[str, Any]:
|
||||
mappings = await _published_mappings(request)
|
||||
search = getattr(request.app.state, "llmwiki_vector_search", None)
|
||||
if search is None:
|
||||
raise HTTPException(status_code=503, detail={"code": "WIKI_INDEX_UNAVAILABLE"})
|
||||
try:
|
||||
payload = await search.search_sections(
|
||||
body.query,
|
||||
mappings,
|
||||
top_k_sections=body.top_k,
|
||||
external=True,
|
||||
)
|
||||
except WikiIndexError as exc:
|
||||
raise HTTPException(
|
||||
status_code=exc.status_code,
|
||||
detail={"code": exc.code, "message": str(exc)},
|
||||
) from None
|
||||
|
||||
# Publication can change while an embedding/search call is in flight. A
|
||||
# second read ensures an unpublished or newly reserved result is never
|
||||
# emitted by this anonymous endpoint.
|
||||
still_published = {str(row["id"]) for row in await _published_mappings(request)}
|
||||
frontend_base_url = request.app.state.config.llmwiki.local_wiki_index.frontend_base_url
|
||||
results: list[dict[str, Any]] = []
|
||||
for raw_item in payload.get("results") or []:
|
||||
if str(raw_item.get("knowledge_base_id") or "") not in still_published:
|
||||
continue
|
||||
item = dict(raw_item)
|
||||
wiki_route, wiki_url = _wiki_link(
|
||||
frontend_base_url,
|
||||
str(item["knowledge_base_id"]),
|
||||
str(item["wiki_slug"]),
|
||||
)
|
||||
item.update(_reference_metadata(item))
|
||||
item["wiki_route"] = wiki_route
|
||||
item["wiki_url"] = wiki_url
|
||||
if not body.include_page_content:
|
||||
item.pop("page_content", None)
|
||||
results.append(item)
|
||||
|
||||
return {
|
||||
"query": body.query,
|
||||
"top_k": body.top_k,
|
||||
"mode": "vector",
|
||||
"results": results,
|
||||
"count": len(results),
|
||||
"knowledge_base_count": payload.get("knowledge_base_count", len(mappings)),
|
||||
"index_fingerprint": payload.get("index_fingerprint"),
|
||||
"partial": bool(payload.get("partial", False)),
|
||||
"warnings": payload.get("warnings") or [],
|
||||
}
|
||||
|
||||
|
||||
__all__ = [
|
||||
"PublicKnowledgeVectorSearchRequest",
|
||||
"public_knowledge_vector_search",
|
||||
"router",
|
||||
]
|
||||
Reference in new issue
Block a user