feat: add compact toolbox and public wiki vector search
This commit is contained in:
1 parent
9f29ecae9e
commit
d35d075669
29 files changed
+757
-128
No files matched your search
@@ -24,8 +24,9 @@ from app.gateway.auth.disabled_mode import is_auth_disabled # noqa: E402
|
||||
from app.gateway.auth_middleware import AuthMiddleware
|
||||
from app.gateway.config import get_gateway_config
|
||||
from app.gateway.csrf_middleware import CSRFMiddleware
|
||||
from app.gateway.deps import langgraph_runtime
|
||||
from app.gateway.routers import (
|
||||
from app.gateway.deps import langgraph_runtime
|
||||
from app.gateway.public_knowledge_cors import PublicKnowledgeCORSMiddleware
|
||||
from app.gateway.routers import (
|
||||
ai_writing,
|
||||
admin_active_runs,
|
||||
admin_users,
|
||||
@@ -66,8 +67,9 @@ from app.gateway.routers import (
|
||||
positions,
|
||||
position_roles,
|
||||
public_config,
|
||||
public_embed,
|
||||
public_leaderboard,
|
||||
public_embed,
|
||||
public_knowledge_vector_search,
|
||||
public_leaderboard,
|
||||
public_scheduled_tasks,
|
||||
recommend,
|
||||
flow_extract,
|
||||
@@ -1473,11 +1475,16 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
"Set GATEWAY_CORS_ORIGINS or GATEWAY_CORS_ALLOW_ALL for deployment.",
|
||||
", ".join(dev_origins),
|
||||
)
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=dev_origins,
|
||||
**cors_kwargs,
|
||||
)
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=dev_origins,
|
||||
**cors_kwargs,
|
||||
)
|
||||
|
||||
# This one anonymous API is intentionally callable by browser code from
|
||||
# every origin. Keep the override route-scoped so authenticated APIs retain
|
||||
# the deployment's normal CORS allow-list.
|
||||
app.add_middleware(PublicKnowledgeCORSMiddleware)
|
||||
|
||||
# Include routers
|
||||
# Models API is mounted at /api/models
|
||||
@@ -1626,8 +1633,11 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
app.include_router(public_config.router)
|
||||
app.include_router(html_page_favorites.router)
|
||||
|
||||
# Product knowledge base API is mounted at /api/knowledge
|
||||
app.include_router(knowledge.router)
|
||||
# Product knowledge base API is mounted at /api/knowledge
|
||||
app.include_router(knowledge.router)
|
||||
# Anonymous, section-level vector search over every published Wiki base
|
||||
# except the system conversation-deposit base.
|
||||
app.include_router(public_knowledge_vector_search.router)
|
||||
|
||||
# External knowledge ingest (publish md artifacts) at /api/knowledge-ingest
|
||||
app.include_router(knowledge_ingest.router)
|
||||
|
||||
@@ -68,9 +68,11 @@ _PUBLIC_EXACT_PATHS: frozenset[str] = frozenset(
|
||||
"/api/v1/auth/login/token",
|
||||
"/api/v1/auth/register",
|
||||
"/api/v1/auth/logout",
|
||||
"/api/v1/auth/setup-status",
|
||||
"/api/v1/auth/initialize",
|
||||
# 并行多智能体面板专用的「绕过常规登录」取 token 端点(内网部署限制了标准登录时用)。
|
||||
"/api/v1/auth/setup-status",
|
||||
"/api/v1/auth/initialize",
|
||||
# Anonymous vector search over published local Wiki indexes.
|
||||
"/api/knowledge/vector-search",
|
||||
# 并行多智能体面板专用的「绕过常规登录」取 token 端点(内网部署限制了标准登录时用)。
|
||||
"/api/parallel-agents/auth/token",
|
||||
}
|
||||
)
|
||||
|
||||
@@ -48,8 +48,10 @@ def should_check_csrf(request: Request) -> bool:
|
||||
path = app_relative_path(request)
|
||||
stripped = path.rstrip("/")
|
||||
# Exempt /api/v1/auth/me endpoint
|
||||
if stripped == "/api/v1/auth/me":
|
||||
return False
|
||||
if stripped == "/api/v1/auth/me":
|
||||
return False
|
||||
if stripped == "/api/knowledge/vector-search":
|
||||
return False
|
||||
# Anonymous public endpoints (mirror auth_middleware._PUBLIC_PATH_PREFIXES).
|
||||
# These intentionally have no cookie session, so the Double-Submit Cookie
|
||||
# pattern is both impossible (no token to compare) and unnecessary (no
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
"""Route-scoped permissive CORS for the anonymous knowledge vector API."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
|
||||
from fastapi import Request, Response
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
from starlette.types import ASGIApp
|
||||
|
||||
from app.gateway.proxy_path import app_relative_path
|
||||
|
||||
PUBLIC_KNOWLEDGE_VECTOR_SEARCH_PATH = "/api/knowledge/vector-search"
|
||||
|
||||
|
||||
def is_public_knowledge_vector_search(request: Request) -> bool:
|
||||
return app_relative_path(request).rstrip("/") == PUBLIC_KNOWLEDGE_VECTOR_SEARCH_PATH
|
||||
|
||||
|
||||
def _apply_public_cors(response: Response) -> Response:
|
||||
response.headers["Access-Control-Allow-Origin"] = "*"
|
||||
response.headers["Access-Control-Allow-Methods"] = "POST, OPTIONS"
|
||||
response.headers["Access-Control-Allow-Headers"] = "*"
|
||||
response.headers["Access-Control-Allow-Private-Network"] = "true"
|
||||
response.headers["Access-Control-Max-Age"] = "600"
|
||||
if "Access-Control-Allow-Credentials" in response.headers:
|
||||
del response.headers["Access-Control-Allow-Credentials"]
|
||||
return response
|
||||
|
||||
|
||||
class PublicKnowledgeCORSMiddleware(BaseHTTPMiddleware):
|
||||
"""Allow every browser origin for one credential-free public route only."""
|
||||
|
||||
def __init__(self, app: ASGIApp) -> None:
|
||||
super().__init__(app)
|
||||
|
||||
async def dispatch(self, request: Request, call_next: Callable) -> Response:
|
||||
if not is_public_knowledge_vector_search(request):
|
||||
return await call_next(request)
|
||||
if request.method.upper() == "OPTIONS":
|
||||
return _apply_public_cors(Response(status_code=204))
|
||||
return _apply_public_cors(await call_next(request))
|
||||
|
||||
|
||||
__all__ = [
|
||||
"PUBLIC_KNOWLEDGE_VECTOR_SEARCH_PATH",
|
||||
"PublicKnowledgeCORSMiddleware",
|
||||
"is_public_knowledge_vector_search",
|
||||
]
|
||||
@@ -17,8 +17,8 @@ from pydantic import BaseModel, Field
|
||||
|
||||
from app.gateway.deps import get_current_user, get_optional_user_from_request
|
||||
from deerflow.persistence.compact_menu_overrides import (
|
||||
CompactMenuOverrideStore,
|
||||
DEFAULT_COMPACT_MENU_TYPE,
|
||||
CompactMenuOverrideStore,
|
||||
normalize_compact_menu_type,
|
||||
)
|
||||
|
||||
@@ -27,7 +27,7 @@ router = APIRouter(prefix="/api/compact-menu-overrides", tags=["compact-menu-ove
|
||||
|
||||
class CompactMenuOverride(BaseModel):
|
||||
node_id: str = Field(..., min_length=1, max_length=128)
|
||||
placement: Literal["sidebar", "settings"] = "sidebar"
|
||||
placement: Literal["sidebar", "settings", "toolbox"] = "sidebar"
|
||||
sort_order: int = 0
|
||||
label: str | None = Field(default=None, max_length=255)
|
||||
|
||||
|
||||
@@ -1151,35 +1151,15 @@ html.deerflow-source-detail-open [data-deerflow-source-detail-panel="true"] [cla
|
||||
if (!proxyUrl) return;
|
||||
if (
|
||||
image.dataset.deerflowProtectedSource === protectedSrc &&
|
||||
String(image.getAttribute("src") || "").startsWith("blob:")
|
||||
String(image.getAttribute("src") || "") === proxyUrl
|
||||
) return;
|
||||
const attempts = Number(image.dataset.deerflowProtectedAttempts || "0");
|
||||
if (attempts >= 3 || image.dataset.deerflowProtectedLoading === "1") return;
|
||||
image.dataset.deerflowProtectedLoading = "1";
|
||||
image.dataset.deerflowProtectedAttempts = String(attempts + 1);
|
||||
nativeFetch(proxyUrl, {{ credentials: "same-origin", headers: {{ Accept: "image/*" }} }})
|
||||
.then((response) => {{
|
||||
if (!response.ok) throw new Error(`protected image request failed (${{response.status}})`);
|
||||
return response.blob();
|
||||
}})
|
||||
.then((blob) => {{
|
||||
const objectUrl = URL.createObjectURL(blob);
|
||||
image.dataset.deerflowProtectedSource = protectedSrc;
|
||||
image.removeAttribute("data-img-loading");
|
||||
image.removeAttribute("data-wiki-image-error");
|
||||
image.src = objectUrl;
|
||||
image.addEventListener("load", () => URL.revokeObjectURL(objectUrl), {{ once: true }});
|
||||
}})
|
||||
.catch(() => {{
|
||||
image.dataset.wikiImageError = "true";
|
||||
window.setTimeout(() => {{
|
||||
delete image.dataset.deerflowProtectedLoading;
|
||||
hydrateProtectedImage(image);
|
||||
}}, 1200 * (attempts + 1));
|
||||
}})
|
||||
.finally(() => {{
|
||||
delete image.dataset.deerflowProtectedLoading;
|
||||
}});
|
||||
// Keep the real HTTP proxy URL on the DOM node. WeKnora's preview/lightbox
|
||||
// reuses the thumbnail's src when the user clicks it; a short-lived blob URL
|
||||
// may render once and then become invalid before the preview opens.
|
||||
image.dataset.deerflowProtectedSource = protectedSrc;
|
||||
image.removeAttribute("data-img-loading");
|
||||
image.removeAttribute("data-wiki-image-error");
|
||||
image.src = proxyUrl;
|
||||
}}
|
||||
function hydrateProtectedImages(scope = document) {{
|
||||
if (scope?.matches?.("img[data-protected-src]")) hydrateProtectedImage(scope);
|
||||
@@ -1197,7 +1177,7 @@ html.deerflow-source-detail-open [data-deerflow-source-detail-panel="true"] [cla
|
||||
childList: true,
|
||||
subtree: true,
|
||||
attributes: true,
|
||||
attributeFilter: ["data-protected-src"]
|
||||
attributeFilter: ["data-protected-src", "src"]
|
||||
}});
|
||||
hydrateProtectedImages();
|
||||
|
||||
|
||||
+152
@@ -0,0 +1,152 @@
|
||||
"""Anonymous vector search over published, locally indexed WeKnora Wiki data."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from urllib.parse import quote, urlencode
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from deerflow.integrations.weknora.local_index.search import WikiIndexError
|
||||
|
||||
router = APIRouter(tags=["public-knowledge-vector-search"])
|
||||
|
||||
|
||||
class PublicKnowledgeVectorSearchRequest(BaseModel):
|
||||
query: str = Field(..., min_length=1, max_length=10_000)
|
||||
top_k: int = Field(default=10, ge=1, le=100)
|
||||
include_page_content: bool = False
|
||||
|
||||
|
||||
def _is_conversation_deposit(mapping: dict[str, Any]) -> bool:
|
||||
return str(mapping.get("owner_user_id") or "") == "system" and str(mapping.get("name") or "").strip() == "对话沉淀"
|
||||
|
||||
|
||||
async def _published_mappings(request: Request) -> list[dict[str, Any]]:
|
||||
store = getattr(request.app.state, "llmwiki_store", None)
|
||||
if store is None:
|
||||
raise HTTPException(status_code=503, detail={"code": "LLMWIKI_STORE_UNAVAILABLE"})
|
||||
rows = await store.list_visible("system", scope="public", is_admin=False)
|
||||
return [row for row in rows if row.get("publication_status") == "published" and not _is_conversation_deposit(row)]
|
||||
|
||||
|
||||
def _unique_strings(values: list[str]) -> list[str]:
|
||||
return list(dict.fromkeys(value for value in values if value))
|
||||
|
||||
|
||||
def _reference_metadata(item: dict[str, Any]) -> dict[str, Any]:
|
||||
source_refs = item.get("source_refs") if isinstance(item.get("source_refs"), list) else []
|
||||
chunk_refs = item.get("chunk_refs") if isinstance(item.get("chunk_refs"), list) else []
|
||||
document_ids: list[str] = []
|
||||
document_names: list[str] = []
|
||||
chunk_ids: list[str] = []
|
||||
source_ref_ids: list[str] = []
|
||||
|
||||
for ref in source_refs:
|
||||
if isinstance(ref, dict):
|
||||
document_id = str(ref.get("document_id") or ref.get("knowledge_id") or ref.get("documentId") or ref.get("knowledgeId") or "").strip()
|
||||
document_name = str(
|
||||
ref.get("document_name") or ref.get("knowledge_title") or ref.get("documentName") or ref.get("knowledgeTitle") or ref.get("file_name") or ref.get("filename") or ref.get("title") or ref.get("name") or ""
|
||||
).strip()
|
||||
chunk_id = str(ref.get("chunk_id") or ref.get("chunkId") or "").strip()
|
||||
if document_id:
|
||||
document_ids.append(document_id)
|
||||
if document_name:
|
||||
document_names.append(document_name)
|
||||
if chunk_id:
|
||||
chunk_ids.append(chunk_id)
|
||||
elif isinstance(ref, (str, int)) and str(ref).strip():
|
||||
source_ref_ids.append(str(ref).strip())
|
||||
|
||||
for ref in chunk_refs:
|
||||
if isinstance(ref, dict):
|
||||
chunk_id = str(ref.get("chunk_id") or ref.get("chunkId") or ref.get("id") or "").strip()
|
||||
else:
|
||||
chunk_id = str(ref or "").strip()
|
||||
if chunk_id:
|
||||
chunk_ids.append(chunk_id)
|
||||
|
||||
document_ids = _unique_strings(document_ids)
|
||||
document_names = _unique_strings(document_names)
|
||||
chunk_ids = _unique_strings(chunk_ids)
|
||||
source_ref_ids = _unique_strings(source_ref_ids)
|
||||
return {
|
||||
"document_id": document_ids[0] if document_ids else None,
|
||||
"document_name": document_names[0] if document_names else None,
|
||||
"document_ids": document_ids,
|
||||
"document_names": document_names,
|
||||
"chunk_id": chunk_ids[0] if chunk_ids else None,
|
||||
"chunk_ids": chunk_ids,
|
||||
"source_ref_ids": source_ref_ids,
|
||||
}
|
||||
|
||||
|
||||
def _wiki_link(frontend_base_url: str, mapping_id: str, wiki_slug: str) -> tuple[str, str]:
|
||||
query = urlencode({"wikiSlug": wiki_slug})
|
||||
route = f"/#/page/workspace/knowledge/{quote(mapping_id, safe='')}/wiki/view?{query}"
|
||||
return route, f"{frontend_base_url.rstrip('/')}{route}"
|
||||
|
||||
|
||||
@router.post("/api/knowledge/vector-search")
|
||||
async def public_knowledge_vector_search(
|
||||
request: Request,
|
||||
body: PublicKnowledgeVectorSearchRequest,
|
||||
) -> dict[str, Any]:
|
||||
mappings = await _published_mappings(request)
|
||||
search = getattr(request.app.state, "llmwiki_vector_search", None)
|
||||
if search is None:
|
||||
raise HTTPException(status_code=503, detail={"code": "WIKI_INDEX_UNAVAILABLE"})
|
||||
try:
|
||||
payload = await search.search_sections(
|
||||
body.query,
|
||||
mappings,
|
||||
top_k_sections=body.top_k,
|
||||
external=True,
|
||||
)
|
||||
except WikiIndexError as exc:
|
||||
raise HTTPException(
|
||||
status_code=exc.status_code,
|
||||
detail={"code": exc.code, "message": str(exc)},
|
||||
) from None
|
||||
|
||||
# Publication can change while an embedding/search call is in flight. A
|
||||
# second read ensures an unpublished or newly reserved result is never
|
||||
# emitted by this anonymous endpoint.
|
||||
still_published = {str(row["id"]) for row in await _published_mappings(request)}
|
||||
frontend_base_url = request.app.state.config.llmwiki.local_wiki_index.frontend_base_url
|
||||
results: list[dict[str, Any]] = []
|
||||
for raw_item in payload.get("results") or []:
|
||||
if str(raw_item.get("knowledge_base_id") or "") not in still_published:
|
||||
continue
|
||||
item = dict(raw_item)
|
||||
wiki_route, wiki_url = _wiki_link(
|
||||
frontend_base_url,
|
||||
str(item["knowledge_base_id"]),
|
||||
str(item["wiki_slug"]),
|
||||
)
|
||||
item.update(_reference_metadata(item))
|
||||
item["wiki_route"] = wiki_route
|
||||
item["wiki_url"] = wiki_url
|
||||
if not body.include_page_content:
|
||||
item.pop("page_content", None)
|
||||
results.append(item)
|
||||
|
||||
return {
|
||||
"query": body.query,
|
||||
"top_k": body.top_k,
|
||||
"mode": "vector",
|
||||
"results": results,
|
||||
"count": len(results),
|
||||
"knowledge_base_count": payload.get("knowledge_base_count", len(mappings)),
|
||||
"index_fingerprint": payload.get("index_fingerprint"),
|
||||
"partial": bool(payload.get("partial", False)),
|
||||
"warnings": payload.get("warnings") or [],
|
||||
}
|
||||
|
||||
|
||||
__all__ = [
|
||||
"PublicKnowledgeVectorSearchRequest",
|
||||
"public_knowledge_vector_search",
|
||||
"router",
|
||||
]
|
||||
Reference in new issue
Block a user