feat: add artifact library and improve public wiki
This commit is contained in:
1 parent
1b43205c1b
commit
dd0ee282df
16 files changed
+4081
-98
No files matched your search
@@ -55,6 +55,7 @@ from deerflow.config.agents_config import load_agent_config
|
||||
from deerflow.config.llmwiki_config import LlmWikiRuntimeOverride, normalize_service_url
|
||||
from deerflow.config.system_settings import load_system_settings, save_system_settings
|
||||
from deerflow.integrations.weknora.client import WeKnoraClient, WeKnoraError
|
||||
from deerflow.integrations.weknora.local_index.retrieval import is_fully_vectorized
|
||||
from deerflow.integrations.weknora.local_index.search import WikiIndexError
|
||||
from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
|
||||
from deerflow.persistence.llmwiki import LlmWikiStore
|
||||
@@ -496,15 +497,7 @@ def _should_follow_weknora_binary_redirect(path: str) -> bool:
|
||||
"""Keep protected preview/image bytes inside the DeerFlow proxy."""
|
||||
|
||||
segments = _path_segments(path)
|
||||
return (
|
||||
len(segments) == 5
|
||||
and segments[:3] == ["api", "v1", "knowledge"]
|
||||
and segments[4] == "preview"
|
||||
) or (
|
||||
len(segments) == 5
|
||||
and segments[:3] == ["api", "v1", "knowledge-bases"]
|
||||
and segments[4] == "files"
|
||||
)
|
||||
return (len(segments) == 5 and segments[:3] == ["api", "v1", "knowledge"] and segments[4] == "preview") or (len(segments) == 5 and segments[:3] == ["api", "v1", "knowledge-bases"] and segments[4] == "files")
|
||||
|
||||
|
||||
def _forward_request_headers(request: Request) -> dict[str, str]:
|
||||
@@ -1833,6 +1826,9 @@ def _mapping_view(row: dict[str, Any], remote: dict[str, Any] | None, *, actor_u
|
||||
"knowledge_count": int(source.get("knowledge_count") or 0),
|
||||
"chunk_count": int(source.get("chunk_count") or 0),
|
||||
"processing_count": int(source.get("processing_count") or 0),
|
||||
"share_count": int(source.get("share_count") or 0),
|
||||
"is_processing": bool(source.get("is_processing") or int(source.get("processing_count") or 0) > 0),
|
||||
"is_temporary": bool(source.get("is_temporary", False)),
|
||||
"embedding_model_id": source.get("embedding_model_id"),
|
||||
"vector_store_id": source.get("vector_store_id"),
|
||||
"vector_store_name": source.get("vector_store_name"),
|
||||
@@ -1840,16 +1836,23 @@ def _mapping_view(row: dict[str, Any], remote: dict[str, Any] | None, *, actor_u
|
||||
"vector_store_engine_type": source.get("vector_store_engine_type"),
|
||||
"vector_store_status": source.get("vector_store_status"),
|
||||
"chunking_config": source.get("chunking_config") if isinstance(source.get("chunking_config"), dict) else {},
|
||||
"image_processing_config": (source.get("image_processing_config") if isinstance(source.get("image_processing_config"), dict) else {}),
|
||||
"indexing_strategy": source.get("indexing_strategy") if isinstance(source.get("indexing_strategy"), dict) else {},
|
||||
"capabilities": source.get("capabilities") if isinstance(source.get("capabilities"), dict) else {},
|
||||
"extract_config": source.get("extract_config") if isinstance(source.get("extract_config"), dict) else {},
|
||||
"faq_config": source.get("faq_config") if isinstance(source.get("faq_config"), dict) else {},
|
||||
"wiki_config": source.get("wiki_config") if isinstance(source.get("wiki_config"), dict) else {},
|
||||
"question_generation_config": (source.get("question_generation_config") if isinstance(source.get("question_generation_config"), dict) else {}),
|
||||
"auto_tag_config": source.get("auto_tag_config") if isinstance(source.get("auto_tag_config"), dict) else {},
|
||||
"vlm_config": source.get("vlm_config") if isinstance(source.get("vlm_config"), dict) else {},
|
||||
"asr_config": source.get("asr_config") if isinstance(source.get("asr_config"), dict) else {},
|
||||
"summary_model_id": source.get("summary_model_id"),
|
||||
"storage_provider": (source.get("storage_provider_config", {}).get("provider") if isinstance(source.get("storage_provider_config"), dict) else None),
|
||||
"created_at": row.get("created_at"),
|
||||
"updated_at": row.get("updated_at"),
|
||||
"published_at": row.get("published_at"),
|
||||
"remote_created_at": source.get("created_at"),
|
||||
"remote_updated_at": source.get("updated_at"),
|
||||
}
|
||||
|
||||
|
||||
@@ -1877,11 +1880,7 @@ async def _public_mapping(request: Request, mapping_id: str) -> dict[str, Any]:
|
||||
write=False,
|
||||
is_admin=False,
|
||||
)
|
||||
if (
|
||||
row is None
|
||||
or row.get("publication_status") != "published"
|
||||
or _is_conversation_deposit_mapping(row)
|
||||
):
|
||||
if row is None or row.get("publication_status") != "published" or _is_conversation_deposit_mapping(row):
|
||||
raise HTTPException(status_code=404, detail="Published knowledge base not found")
|
||||
return row
|
||||
|
||||
@@ -2537,24 +2536,260 @@ async def test_llmwiki_connection(request: Request, body: LlmWikiConnectionTest)
|
||||
return {"success": True, "provider": "weknora", "health": health}
|
||||
|
||||
|
||||
_PUBLIC_WIKI_SUMMARY_CONCURRENCY = 8
|
||||
_PUBLIC_WIKI_PAGE_SIZE = 500
|
||||
_PUBLIC_WIKI_PAGE_LIMIT = 100
|
||||
_GENERATED_PUBLIC_WIKI_INDEX_SLUG = "__index__"
|
||||
|
||||
|
||||
def _is_public_wiki_index_page(page: dict[str, Any]) -> bool:
|
||||
page_type = str(page.get("page_type") or "").strip().lower()
|
||||
title = str(page.get("title") or "").strip().lower()
|
||||
slug_name = str(page.get("slug") or "").strip("/").rsplit("/", 1)[-1].lower()
|
||||
return page_type == "index" or title in {"索引", "index"} or slug_name in {"index", "home", "readme"}
|
||||
|
||||
|
||||
def _public_wiki_default_page(pages: list[dict[str, Any]]) -> dict[str, Any] | None:
|
||||
return (
|
||||
next((page for page in pages if _is_public_wiki_index_page(page)), None)
|
||||
or next((page for page in pages if str(page.get("page_type") or "").lower() == "overview"), None)
|
||||
or next(
|
||||
(page for page in pages if not str(page.get("parent_slug") or "").strip() and int(page.get("depth") or 0) == 0),
|
||||
None,
|
||||
)
|
||||
or (pages[0] if pages else None)
|
||||
)
|
||||
|
||||
|
||||
def _count_public_wiki_values(pages: list[dict[str, Any]], key: str, fallback: str) -> dict[str, int]:
|
||||
counts: dict[str, int] = {}
|
||||
for page in pages:
|
||||
value = str(page.get(key) or fallback).strip().lower() or fallback
|
||||
counts[value] = counts.get(value, 0) + 1
|
||||
return counts
|
||||
|
||||
|
||||
def _public_wiki_time_bound(pages: list[dict[str, Any]], key: str, *, latest: bool) -> str | None:
|
||||
values = [str(page.get(key) or "").strip() for page in pages]
|
||||
values = [value for value in values if value]
|
||||
if not values:
|
||||
return None
|
||||
return max(values) if latest else min(values)
|
||||
|
||||
|
||||
async def _all_public_wiki_page_views(
|
||||
client: WeKnoraClient,
|
||||
remote_id: str,
|
||||
) -> tuple[list[dict[str, Any]], int, bool]:
|
||||
pages_by_slug: dict[str, dict[str, Any]] = {}
|
||||
remote_total = 0
|
||||
complete = False
|
||||
for page_number in range(1, _PUBLIC_WIKI_PAGE_LIMIT + 1):
|
||||
result = await client.list_wiki_pages(
|
||||
remote_id,
|
||||
page=page_number,
|
||||
page_size=_PUBLIC_WIKI_PAGE_SIZE,
|
||||
query="",
|
||||
)
|
||||
raw_pages = result.get("pages") if isinstance(result, dict) else []
|
||||
page_items = raw_pages if isinstance(raw_pages, list) else []
|
||||
for page_item in page_items:
|
||||
if not isinstance(page_item, dict):
|
||||
continue
|
||||
page = _wiki_page_view(page_item)
|
||||
if page is None:
|
||||
continue
|
||||
slug = str(page.get("slug") or "").strip()
|
||||
pages_by_slug[slug or f"__page_{len(pages_by_slug)}"] = page
|
||||
reported_total = int(result.get("total") or 0)
|
||||
remote_total = max(remote_total, reported_total, len(pages_by_slug))
|
||||
total_pages = max(1, int(result.get("total_pages") or 1))
|
||||
if not page_items or page_number >= total_pages or (reported_total > 0 and len(pages_by_slug) >= remote_total):
|
||||
complete = True
|
||||
break
|
||||
return list(pages_by_slug.values()), max(remote_total, len(pages_by_slug)), complete
|
||||
|
||||
|
||||
def _public_wiki_summary(
|
||||
mapping_id: str,
|
||||
pages: list[dict[str, Any]],
|
||||
*,
|
||||
total: int,
|
||||
complete: bool,
|
||||
) -> dict[str, Any]:
|
||||
explicit_index = next((page for page in pages if _is_public_wiki_index_page(page)), None)
|
||||
default_page = _public_wiki_default_page(pages)
|
||||
if explicit_index is not None:
|
||||
index_mode = "explicit"
|
||||
index_slug: str | None = str(explicit_index.get("slug") or "") or None
|
||||
elif pages:
|
||||
index_mode = "generated"
|
||||
index_slug = _GENERATED_PUBLIC_WIKI_INDEX_SLUG
|
||||
else:
|
||||
index_mode = "none"
|
||||
index_slug = None
|
||||
default_slug = index_slug or (str(default_page.get("slug") or "") if default_page else None)
|
||||
route = f"/embed/knowledge/{quote(mapping_id, safe='')}"
|
||||
if default_slug:
|
||||
route = f"{route}?{urlencode({'wikiSlug': default_slug})}"
|
||||
return {
|
||||
"status": "ready" if pages else "empty",
|
||||
"page_count": total,
|
||||
"loaded_page_count": len(pages),
|
||||
"stats_complete": complete,
|
||||
"page_type_counts": _count_public_wiki_values(pages, "page_type", "page"),
|
||||
"page_status_counts": _count_public_wiki_values(pages, "status", "unknown"),
|
||||
"index_mode": index_mode,
|
||||
"has_explicit_index": explicit_index is not None,
|
||||
"index_slug": index_slug,
|
||||
"default_slug": default_slug,
|
||||
"created_at": _public_wiki_time_bound(pages, "created_at", latest=False),
|
||||
"updated_at": _public_wiki_time_bound(pages, "updated_at", latest=True),
|
||||
"embed_route": route,
|
||||
}
|
||||
|
||||
|
||||
def _unavailable_public_wiki_summary(mapping_id: str) -> dict[str, Any]:
|
||||
return {
|
||||
"status": "unavailable",
|
||||
"page_count": 0,
|
||||
"loaded_page_count": 0,
|
||||
"stats_complete": False,
|
||||
"page_type_counts": {},
|
||||
"page_status_counts": {},
|
||||
"index_mode": "unknown",
|
||||
"has_explicit_index": False,
|
||||
"index_slug": None,
|
||||
"default_slug": None,
|
||||
"created_at": None,
|
||||
"updated_at": None,
|
||||
"embed_route": f"/embed/knowledge/{quote(mapping_id, safe='')}",
|
||||
}
|
||||
|
||||
|
||||
def _public_local_index_summary(
|
||||
request: Request,
|
||||
mapping: dict[str, Any],
|
||||
state: dict[str, Any] | None,
|
||||
) -> dict[str, Any]:
|
||||
app_state = request.app.state
|
||||
local_config = getattr(
|
||||
getattr(getattr(app_state, "config", None), "llmwiki", None),
|
||||
"local_wiki_index",
|
||||
None,
|
||||
)
|
||||
enabled = bool(getattr(local_config, "enabled", False))
|
||||
current = dict(state or {})
|
||||
state_name = str(current.get("state") or "not_synced")
|
||||
if state_name == "idle" and int(current.get("index_revision") or 0) == 0 and not current.get("last_completed_at") and not current.get("last_success_at"):
|
||||
state_name = "not_synced"
|
||||
embedding = getattr(app_state, "llmwiki_embedding", None)
|
||||
fingerprint = embedding.fingerprint if embedding is not None else None
|
||||
fully_vectorized = bool(mapping.get("wiki_index_enabled", True)) and is_fully_vectorized(
|
||||
current,
|
||||
fingerprint,
|
||||
)
|
||||
return {
|
||||
"enabled": enabled,
|
||||
"wiki_index_enabled": bool(mapping.get("wiki_index_enabled", True)),
|
||||
"state": state_name,
|
||||
"fully_vectorized": fully_vectorized,
|
||||
"retrieval_mode": "vector" if fully_vectorized else "wiki_api",
|
||||
"remote_page_count": int(current.get("remote_page_count") or 0),
|
||||
"local_page_count": int(current.get("local_page_count") or 0),
|
||||
"ready_page_count": int(current.get("ready_page_count") or 0),
|
||||
"failed_page_count": int(current.get("failed_page_count") or 0),
|
||||
"vector_count": int(current.get("vector_count") or 0),
|
||||
"last_started_at": current.get("last_started_at"),
|
||||
"last_completed_at": current.get("last_completed_at"),
|
||||
"last_success_at": current.get("last_success_at"),
|
||||
"updated_at": current.get("updated_at"),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/api/public/llmwiki/knowledge-bases")
|
||||
async def list_public_knowledge_bases(request: Request) -> dict[str, Any]:
|
||||
"""List the minimal identity of every knowledge base published as a public Wiki."""
|
||||
"""List live, published Wiki libraries with safe metadata and index health."""
|
||||
|
||||
rows = await _store(request).list_visible(
|
||||
"__public_wiki__",
|
||||
scope="public",
|
||||
is_admin=False,
|
||||
)
|
||||
knowledge_bases = [
|
||||
{
|
||||
"id": str(row["id"]),
|
||||
"name": str(row.get("name") or ""),
|
||||
}
|
||||
for row in rows
|
||||
if row.get("publication_status") == "published"
|
||||
and not _is_conversation_deposit_mapping(row)
|
||||
rows = [
|
||||
row
|
||||
for row in await _store(request).list_visible(
|
||||
"__public_wiki__",
|
||||
scope="public",
|
||||
is_admin=False,
|
||||
)
|
||||
if row.get("publication_status") == "published" and not _is_conversation_deposit_mapping(row)
|
||||
]
|
||||
client = _client_or_503()
|
||||
try:
|
||||
remote_items = await client.list_knowledge_bases()
|
||||
except WeKnoraError as exc:
|
||||
_raise_upstream(exc)
|
||||
remote_by_id = {str(item.get("id")): item for item in remote_items if isinstance(item, dict) and item.get("id")}
|
||||
# A successful upstream list is authoritative: stale DeerFlow mappings are
|
||||
# omitted so discovery never advertises a Wiki that its detail route cannot open.
|
||||
rows = [row for row in rows if str(row.get("weknora_id") or "") in remote_by_id]
|
||||
mapping_ids = [str(row["id"]) for row in rows]
|
||||
index_states: dict[str, dict[str, Any]] = {}
|
||||
index_store = getattr(request.app.state, "llmwiki_index_store", None)
|
||||
if index_store is not None and mapping_ids:
|
||||
try:
|
||||
index_states = {str(state.get("knowledge_base_mapping_id") or ""): state for state in await index_store.list_index_status(mapping_ids) if isinstance(state, dict)}
|
||||
except Exception:
|
||||
logger.warning("Failed to read public Wiki local-index summaries", exc_info=True)
|
||||
|
||||
semaphore = asyncio.Semaphore(_PUBLIC_WIKI_SUMMARY_CONCURRENCY)
|
||||
|
||||
async def build_summary(row: dict[str, Any]) -> dict[str, Any]:
|
||||
mapping_id = str(row["id"])
|
||||
remote_id = str(row["weknora_id"])
|
||||
remote = remote_by_id[remote_id]
|
||||
async with semaphore:
|
||||
try:
|
||||
remote = {**remote, **await client.get_knowledge_base(remote_id)}
|
||||
except WeKnoraError as exc:
|
||||
if exc.status_code == 404:
|
||||
return {}
|
||||
logger.warning(
|
||||
"Failed to load public Wiki detail metadata: mapping_id=%s",
|
||||
mapping_id,
|
||||
exc_info=True,
|
||||
)
|
||||
try:
|
||||
pages, page_total, complete = await _all_public_wiki_page_views(client, remote_id)
|
||||
wiki = _public_wiki_summary(
|
||||
mapping_id,
|
||||
pages,
|
||||
total=page_total,
|
||||
complete=complete,
|
||||
)
|
||||
except WeKnoraError as exc:
|
||||
logger.warning(
|
||||
"Failed to load public Wiki page summary: mapping_id=%s",
|
||||
mapping_id,
|
||||
exc_info=True,
|
||||
)
|
||||
wiki = (
|
||||
_public_wiki_summary(mapping_id, [], total=0, complete=True)
|
||||
if exc.status_code == 404
|
||||
else _unavailable_public_wiki_summary(mapping_id)
|
||||
)
|
||||
return {
|
||||
**_mapping_view(
|
||||
row,
|
||||
remote,
|
||||
actor_user_id="__public_wiki__",
|
||||
is_admin=False,
|
||||
),
|
||||
"wiki": wiki,
|
||||
"local_index": _public_local_index_summary(
|
||||
request,
|
||||
row,
|
||||
index_states.get(mapping_id),
|
||||
),
|
||||
}
|
||||
|
||||
knowledge_bases = [summary for summary in await asyncio.gather(*(build_summary(row) for row in rows)) if summary]
|
||||
return {"knowledge_bases": knowledge_bases, "total": len(knowledge_bases)}
|
||||
|
||||
|
||||
@@ -2596,13 +2831,7 @@ async def list_public_wiki_pages(
|
||||
except WeKnoraError as exc:
|
||||
_raise_upstream(exc)
|
||||
raw_pages = result.get("pages") if isinstance(result, dict) else []
|
||||
pages = [
|
||||
view
|
||||
for page_item in (raw_pages if isinstance(raw_pages, list) else [])
|
||||
if isinstance(page_item, dict)
|
||||
for view in [_wiki_page_view(page_item)]
|
||||
if view is not None
|
||||
]
|
||||
pages = [view for page_item in (raw_pages if isinstance(raw_pages, list) else []) if isinstance(page_item, dict) for view in [_wiki_page_view(page_item)] if view is not None]
|
||||
return {**result, "pages": pages}
|
||||
|
||||
|
||||
@@ -2650,11 +2879,7 @@ def _public_wiki_index_item(value: Any) -> dict[str, Any] | None:
|
||||
"slug": slug,
|
||||
"title": str(value.get("title") or slug),
|
||||
"summary": str(value.get("summary") or ""),
|
||||
"category_path": (
|
||||
[str(item) for item in value.get("category_path", []) if str(item).strip()]
|
||||
if isinstance(value.get("category_path"), list)
|
||||
else []
|
||||
),
|
||||
"category_path": ([str(item) for item in value.get("category_path", []) if str(item).strip()] if isinstance(value.get("category_path"), list) else []),
|
||||
"wiki_path": str(value.get("wiki_path") or ""),
|
||||
"depth": int(value.get("depth") or 0),
|
||||
}
|
||||
@@ -2680,20 +2905,12 @@ async def _all_public_wiki_index(client: WeKnoraClient, remote_id: str) -> dict[
|
||||
version = max(version, int(payload.get("version") or 0))
|
||||
raw_groups = payload.get("groups") if isinstance(payload.get("groups"), list) else []
|
||||
group = next(
|
||||
(
|
||||
value
|
||||
for value in raw_groups
|
||||
if isinstance(value, dict) and value.get("type") == page_type
|
||||
),
|
||||
(value for value in raw_groups if isinstance(value, dict) and value.get("type") == page_type),
|
||||
{},
|
||||
)
|
||||
total = max(total, int(group.get("total") or 0))
|
||||
raw_items = group.get("items") if isinstance(group.get("items"), list) else []
|
||||
items.extend(
|
||||
item
|
||||
for value in raw_items
|
||||
if (item := _public_wiki_index_item(value)) is not None
|
||||
)
|
||||
items.extend(item for value in raw_items if (item := _public_wiki_index_item(value)) is not None)
|
||||
next_cursor = str(group.get("next_cursor") or "")
|
||||
if not next_cursor or next_cursor in seen_cursors:
|
||||
break
|
||||
@@ -3201,12 +3418,7 @@ async def search_llmwiki(request: Request, body: SearchRequest) -> dict[str, Any
|
||||
except WikiIndexError as exc:
|
||||
raise HTTPException(status_code=exc.status_code, detail={"code": exc.code, "message": str(exc)}) from None
|
||||
mapping_by_id = {str(row["id"]): row for row in rows}
|
||||
result["results"] = [
|
||||
_sanitize_conversation_deposit_value(item)
|
||||
if _is_conversation_deposit_mapping(mapping_by_id.get(str(item.get("knowledge_base_id") or ""), {}))
|
||||
else item
|
||||
for item in result.get("results") or []
|
||||
]
|
||||
result["results"] = [_sanitize_conversation_deposit_value(item) if _is_conversation_deposit_mapping(mapping_by_id.get(str(item.get("knowledge_base_id") or ""), {})) else item for item in result.get("results") or []]
|
||||
return result
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user