feat: add compact toolbox and public wiki vector search

This commit is contained in:
吉鑫升 committed 2026-08-26 14:40:57 +08:00
1 parent 9f29ecae9e
commit d35d075669
29 files changed
+757 -128

No files matched your search

@@ -24,8 +24,9 @@ from app.gateway.auth.disabled_mode import is_auth_disabled # noqa: E402
from app.gateway.auth_middleware import AuthMiddleware
from app.gateway.config import get_gateway_config
from app.gateway.csrf_middleware import CSRFMiddleware
from app.gateway.deps import langgraph_runtime
from app.gateway.routers import (
from app.gateway.deps import langgraph_runtime
from app.gateway.public_knowledge_cors import PublicKnowledgeCORSMiddleware
from app.gateway.routers import (
ai_writing,
admin_active_runs,
admin_users,
@@ -66,8 +67,9 @@ from app.gateway.routers import (
positions,
position_roles,
public_config,
public_embed,
public_leaderboard,
public_embed,
public_knowledge_vector_search,
public_leaderboard,
public_scheduled_tasks,
recommend,
flow_extract,
@@ -1473,11 +1475,16 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
"Set GATEWAY_CORS_ORIGINS or GATEWAY_CORS_ALLOW_ALL for deployment.",
", ".join(dev_origins),
)
app.add_middleware(
CORSMiddleware,
allow_origins=dev_origins,
**cors_kwargs,
)
app.add_middleware(
CORSMiddleware,
allow_origins=dev_origins,
**cors_kwargs,
)
# This one anonymous API is intentionally callable by browser code from
# every origin. Keep the override route-scoped so authenticated APIs retain
# the deployment's normal CORS allow-list.
app.add_middleware(PublicKnowledgeCORSMiddleware)
# Include routers
# Models API is mounted at /api/models
@@ -1626,8 +1633,11 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
app.include_router(public_config.router)
app.include_router(html_page_favorites.router)
# Product knowledge base API is mounted at /api/knowledge
app.include_router(knowledge.router)
# Product knowledge base API is mounted at /api/knowledge
app.include_router(knowledge.router)
# Anonymous, section-level vector search over every published Wiki base
# except the system conversation-deposit base.
app.include_router(public_knowledge_vector_search.router)
# External knowledge ingest (publish md artifacts) at /api/knowledge-ingest
app.include_router(knowledge_ingest.router)
@@ -68,9 +68,11 @@ _PUBLIC_EXACT_PATHS: frozenset[str] = frozenset(
"/api/v1/auth/login/token",
"/api/v1/auth/register",
"/api/v1/auth/logout",
"/api/v1/auth/setup-status",
"/api/v1/auth/initialize",
# 并行多智能体面板专用的「绕过常规登录」取 token 端点(内网部署限制了标准登录时用)。
"/api/v1/auth/setup-status",
"/api/v1/auth/initialize",
# Anonymous vector search over published local Wiki indexes.
"/api/knowledge/vector-search",
# 并行多智能体面板专用的「绕过常规登录」取 token 端点(内网部署限制了标准登录时用)。
"/api/parallel-agents/auth/token",
}
)
@@ -48,8 +48,10 @@ def should_check_csrf(request: Request) -> bool:
path = app_relative_path(request)
stripped = path.rstrip("/")
# Exempt /api/v1/auth/me endpoint
if stripped == "/api/v1/auth/me":
return False
if stripped == "/api/v1/auth/me":
return False
if stripped == "/api/knowledge/vector-search":
return False
# Anonymous public endpoints (mirror auth_middleware._PUBLIC_PATH_PREFIXES).
# These intentionally have no cookie session, so the Double-Submit Cookie
# pattern is both impossible (no token to compare) and unnecessary (no
@@ -0,0 +1,49 @@
"""Route-scoped permissive CORS for the anonymous knowledge vector API."""
from __future__ import annotations
from collections.abc import Callable
from fastapi import Request, Response
from starlette.middleware.base import BaseHTTPMiddleware
from starlette.types import ASGIApp
from app.gateway.proxy_path import app_relative_path
PUBLIC_KNOWLEDGE_VECTOR_SEARCH_PATH = "/api/knowledge/vector-search"
def is_public_knowledge_vector_search(request: Request) -> bool:
return app_relative_path(request).rstrip("/") == PUBLIC_KNOWLEDGE_VECTOR_SEARCH_PATH
def _apply_public_cors(response: Response) -> Response:
response.headers["Access-Control-Allow-Origin"] = "*"
response.headers["Access-Control-Allow-Methods"] = "POST, OPTIONS"
response.headers["Access-Control-Allow-Headers"] = "*"
response.headers["Access-Control-Allow-Private-Network"] = "true"
response.headers["Access-Control-Max-Age"] = "600"
if "Access-Control-Allow-Credentials" in response.headers:
del response.headers["Access-Control-Allow-Credentials"]
return response
class PublicKnowledgeCORSMiddleware(BaseHTTPMiddleware):
"""Allow every browser origin for one credential-free public route only."""
def __init__(self, app: ASGIApp) -> None:
super().__init__(app)
async def dispatch(self, request: Request, call_next: Callable) -> Response:
if not is_public_knowledge_vector_search(request):
return await call_next(request)
if request.method.upper() == "OPTIONS":
return _apply_public_cors(Response(status_code=204))
return _apply_public_cors(await call_next(request))
__all__ = [
"PUBLIC_KNOWLEDGE_VECTOR_SEARCH_PATH",
"PublicKnowledgeCORSMiddleware",
"is_public_knowledge_vector_search",
]
@@ -17,8 +17,8 @@ from pydantic import BaseModel, Field
from app.gateway.deps import get_current_user, get_optional_user_from_request
from deerflow.persistence.compact_menu_overrides import (
CompactMenuOverrideStore,
DEFAULT_COMPACT_MENU_TYPE,
CompactMenuOverrideStore,
normalize_compact_menu_type,
)
@@ -27,7 +27,7 @@ router = APIRouter(prefix="/api/compact-menu-overrides", tags=["compact-menu-ove
class CompactMenuOverride(BaseModel):
node_id: str = Field(..., min_length=1, max_length=128)
placement: Literal["sidebar", "settings"] = "sidebar"
placement: Literal["sidebar", "settings", "toolbox"] = "sidebar"
sort_order: int = 0
label: str | None = Field(default=None, max_length=255)
@@ -1151,35 +1151,15 @@ html.deerflow-source-detail-open [data-deerflow-source-detail-panel="true"] [cla
if (!proxyUrl) return;
if (
image.dataset.deerflowProtectedSource === protectedSrc &&
String(image.getAttribute("src") || "").startsWith("blob:")
String(image.getAttribute("src") || "") === proxyUrl
) return;
const attempts = Number(image.dataset.deerflowProtectedAttempts || "0");
if (attempts >= 3 || image.dataset.deerflowProtectedLoading === "1") return;
image.dataset.deerflowProtectedLoading = "1";
image.dataset.deerflowProtectedAttempts = String(attempts + 1);
nativeFetch(proxyUrl, {{ credentials: "same-origin", headers: {{ Accept: "image/*" }} }})
.then((response) => {{
if (!response.ok) throw new Error(`protected image request failed (${{response.status}})`);
return response.blob();
}})
.then((blob) => {{
const objectUrl = URL.createObjectURL(blob);
image.dataset.deerflowProtectedSource = protectedSrc;
image.removeAttribute("data-img-loading");
image.removeAttribute("data-wiki-image-error");
image.src = objectUrl;
image.addEventListener("load", () => URL.revokeObjectURL(objectUrl), {{ once: true }});
}})
.catch(() => {{
image.dataset.wikiImageError = "true";
window.setTimeout(() => {{
delete image.dataset.deerflowProtectedLoading;
hydrateProtectedImage(image);
}}, 1200 * (attempts + 1));
}})
.finally(() => {{
delete image.dataset.deerflowProtectedLoading;
}});
// Keep the real HTTP proxy URL on the DOM node. WeKnora's preview/lightbox
// reuses the thumbnail's src when the user clicks it; a short-lived blob URL
// may render once and then become invalid before the preview opens.
image.dataset.deerflowProtectedSource = protectedSrc;
image.removeAttribute("data-img-loading");
image.removeAttribute("data-wiki-image-error");
image.src = proxyUrl;
}}
function hydrateProtectedImages(scope = document) {{
if (scope?.matches?.("img[data-protected-src]")) hydrateProtectedImage(scope);
@@ -1197,7 +1177,7 @@ html.deerflow-source-detail-open [data-deerflow-source-detail-panel="true"] [cla
childList: true,
subtree: true,
attributes: true,
attributeFilter: ["data-protected-src"]
attributeFilter: ["data-protected-src", "src"]
}});
hydrateProtectedImages();
@@ -0,0 +1,152 @@
"""Anonymous vector search over published, locally indexed WeKnora Wiki data."""
from __future__ import annotations
from typing import Any
from urllib.parse import quote, urlencode
from fastapi import APIRouter, HTTPException, Request
from pydantic import BaseModel, Field
from deerflow.integrations.weknora.local_index.search import WikiIndexError
router = APIRouter(tags=["public-knowledge-vector-search"])
class PublicKnowledgeVectorSearchRequest(BaseModel):
query: str = Field(..., min_length=1, max_length=10_000)
top_k: int = Field(default=10, ge=1, le=100)
include_page_content: bool = False
def _is_conversation_deposit(mapping: dict[str, Any]) -> bool:
return str(mapping.get("owner_user_id") or "") == "system" and str(mapping.get("name") or "").strip() == "对话沉淀"
async def _published_mappings(request: Request) -> list[dict[str, Any]]:
store = getattr(request.app.state, "llmwiki_store", None)
if store is None:
raise HTTPException(status_code=503, detail={"code": "LLMWIKI_STORE_UNAVAILABLE"})
rows = await store.list_visible("system", scope="public", is_admin=False)
return [row for row in rows if row.get("publication_status") == "published" and not _is_conversation_deposit(row)]
def _unique_strings(values: list[str]) -> list[str]:
return list(dict.fromkeys(value for value in values if value))
def _reference_metadata(item: dict[str, Any]) -> dict[str, Any]:
source_refs = item.get("source_refs") if isinstance(item.get("source_refs"), list) else []
chunk_refs = item.get("chunk_refs") if isinstance(item.get("chunk_refs"), list) else []
document_ids: list[str] = []
document_names: list[str] = []
chunk_ids: list[str] = []
source_ref_ids: list[str] = []
for ref in source_refs:
if isinstance(ref, dict):
document_id = str(ref.get("document_id") or ref.get("knowledge_id") or ref.get("documentId") or ref.get("knowledgeId") or "").strip()
document_name = str(
ref.get("document_name") or ref.get("knowledge_title") or ref.get("documentName") or ref.get("knowledgeTitle") or ref.get("file_name") or ref.get("filename") or ref.get("title") or ref.get("name") or ""
).strip()
chunk_id = str(ref.get("chunk_id") or ref.get("chunkId") or "").strip()
if document_id:
document_ids.append(document_id)
if document_name:
document_names.append(document_name)
if chunk_id:
chunk_ids.append(chunk_id)
elif isinstance(ref, (str, int)) and str(ref).strip():
source_ref_ids.append(str(ref).strip())
for ref in chunk_refs:
if isinstance(ref, dict):
chunk_id = str(ref.get("chunk_id") or ref.get("chunkId") or ref.get("id") or "").strip()
else:
chunk_id = str(ref or "").strip()
if chunk_id:
chunk_ids.append(chunk_id)
document_ids = _unique_strings(document_ids)
document_names = _unique_strings(document_names)
chunk_ids = _unique_strings(chunk_ids)
source_ref_ids = _unique_strings(source_ref_ids)
return {
"document_id": document_ids[0] if document_ids else None,
"document_name": document_names[0] if document_names else None,
"document_ids": document_ids,
"document_names": document_names,
"chunk_id": chunk_ids[0] if chunk_ids else None,
"chunk_ids": chunk_ids,
"source_ref_ids": source_ref_ids,
}
def _wiki_link(frontend_base_url: str, mapping_id: str, wiki_slug: str) -> tuple[str, str]:
query = urlencode({"wikiSlug": wiki_slug})
route = f"/#/page/workspace/knowledge/{quote(mapping_id, safe='')}/wiki/view?{query}"
return route, f"{frontend_base_url.rstrip('/')}{route}"
@router.post("/api/knowledge/vector-search")
async def public_knowledge_vector_search(
request: Request,
body: PublicKnowledgeVectorSearchRequest,
) -> dict[str, Any]:
mappings = await _published_mappings(request)
search = getattr(request.app.state, "llmwiki_vector_search", None)
if search is None:
raise HTTPException(status_code=503, detail={"code": "WIKI_INDEX_UNAVAILABLE"})
try:
payload = await search.search_sections(
body.query,
mappings,
top_k_sections=body.top_k,
external=True,
)
except WikiIndexError as exc:
raise HTTPException(
status_code=exc.status_code,
detail={"code": exc.code, "message": str(exc)},
) from None
# Publication can change while an embedding/search call is in flight. A
# second read ensures an unpublished or newly reserved result is never
# emitted by this anonymous endpoint.
still_published = {str(row["id"]) for row in await _published_mappings(request)}
frontend_base_url = request.app.state.config.llmwiki.local_wiki_index.frontend_base_url
results: list[dict[str, Any]] = []
for raw_item in payload.get("results") or []:
if str(raw_item.get("knowledge_base_id") or "") not in still_published:
continue
item = dict(raw_item)
wiki_route, wiki_url = _wiki_link(
frontend_base_url,
str(item["knowledge_base_id"]),
str(item["wiki_slug"]),
)
item.update(_reference_metadata(item))
item["wiki_route"] = wiki_route
item["wiki_url"] = wiki_url
if not body.include_page_content:
item.pop("page_content", None)
results.append(item)
return {
"query": body.query,
"top_k": body.top_k,
"mode": "vector",
"results": results,
"count": len(results),
"knowledge_base_count": payload.get("knowledge_base_count", len(mappings)),
"index_fingerprint": payload.get("index_fingerprint"),
"partial": bool(payload.get("partial", False)),
"warnings": payload.get("warnings") or [],
}
__all__ = [
"PublicKnowledgeVectorSearchRequest",
"public_knowledge_vector_search",
"router",
]