feat: add artifact library and improve public wiki
This commit is contained in:
1 parent
1b43205c1b
commit
dd0ee282df
16 files changed
+4081
-98
No files matched your search
@@ -321,7 +321,7 @@ FastAPI application on port 8001 with health checks at `GET|HEAD /health`, `/api
|
||||
| **Uploads** (`/api/threads/{id}/uploads`) | `POST /` - upload files (auto-converts PDF/PPT/Excel/Word); `GET /list` - list; `DELETE /{filename}` - delete |
|
||||
| **Threads** (`/api/threads/{id}`) | `DELETE /` - remove DeerFlow-managed local thread data after LangGraph thread deletion; unexpected failures are logged server-side and return a generic 500 detail |
|
||||
| **Threads search** (`POST /threads/search`, LangGraph-compat) | Conversation listing. **System threads are excluded by default**: rows whose `metadata.system` is truthy (scheduler runs, roundtable workers) never crowd the user's list; a caller whose `metadata` filter explicitly references `thread_type`/`system` (e.g. studio notebooks) opts out of the exclusion. New `query` field = case-insensitive title (`display_name`) substring search, used by the chats page's server-side pagination. `POST /threads/count` takes the same request body (limit/offset ignored) and returns `{total}` under identical filtering/exclusion semantics ? powers the chats page's ??/??? display without fetching rows (`ThreadMetaStore.count`). `ThreadMetaStore.search` gained `exclude_system`/`query` params; JSON-blob filters batch-scan until the requested `limit`/`offset` window is full (so pagination over a heavily-filtered set never under-fills), with a `thread_id` tie-breaker on the `updated_at DESC` ordering for stable OFFSET paging. Tests: `tests/test_thread_meta_search.py` |
|
||||
| **Artifacts** (`/api/threads/{id}/artifacts`) | `GET /{path}` - serve artifacts; active content types (`text/html`, `application/xhtml+xml`, `image/svg+xml`) are always forced as download attachments to reduce XSS risk; `?download=true` still forces download for other file types |
|
||||
| **Artifacts** (`/api/threads/{id}/artifacts`, `/api/artifact-library`) | `GET /{path}` - serve artifacts; active content types (`text/html`, `application/xhtml+xml`, `image/svg+xml`) are always forced as download attachments to reduce XSS risk; `?download=true` still forces download for other file types. `GET /api/artifact-library` scans every current-user normal thread's authoritative `outputs/` directory (including files omitted from `present_files`), excludes uploads and image/audio/video media, joins the owning conversation title/agent route metadata, and exposes search/filter/pagination with a bounded 15-second per-user cache plus `refresh=true` bypass. Frontend route `/page/workspace/library` links each item back to the source chat with `?artifact=<path>` so the chat opens the existing artifact panel directly. |
|
||||
| **Word export** (`POST /api/writing/export/docx`) | Shared Markdown → DOCX generator used by conversation, artifact, scheduled-task, Deep Research, and AI-writing download surfaces. `app/gateway/word_export.py` builds OOXML with no Python conversion dependency: A4 portrait, 3.7/3.5/2.8/2.6 cm margins, real multilevel heading numbering, formal body/table styles, page numbers beginning at 1 with odd pages left-aligned and even pages right-aligned, and embedded `app/gateway/assets/fonts/方正小标宋简体.ttf`. The font loader validates its internal family, SHA-256, and OpenType embedding permission (`OS/2.fsType`) before export. Tests: `tests/test_word_export.py`. |
|
||||
| **Deep Research** (`/api/deep-research`) | Durable, user-owned research sessions/jobs with replayable SSE events and hidden-thread artifacts. `basic`/`quick` use the compact runner; `detailed` creates bounded subtopics, searches/writes sections concurrently, and assembles a report; `deep` recursively searches follow-up questions under a deterministic breadth/depth query budget. `multi_agent` is a LangGraph 1.x editor/researcher/writer/reviewer graph: its initial plan emits `awaiting_input`, the recorder stores the interrupt and releases the lease, and `POST /jobs/{id}/resume` rebuilds the graph from the persisted approved/revised plan. A bounded `custom_outline` is frozen into the session; headings/list entries deterministically supply the detailed and multi-agent plan, and the shared writer receives it only as a guarded content constraint. Completed reports support owner-scoped follow-up (`GET /sessions/{id}/messages`, `POST /sessions/{id}/chat`): context is only the current report, selected sources and recent messages; source-id citations are persisted, while `allow_new_research=true` is explicitly rejected. Owners may change that evidence set only after the job stops via `PATCH /sessions/{id}/sources/{source_id}`; this applies to future follow-ups rather than rewriting a completed report. Optional report illustrations are gated by `config.yaml -> deep_research.image_generation`: the Harness `OpenAICompatibleImageProvider` requires base64 output, the shared runner step writes only bounded `research-image-01..04.(png|jpg|webp)` artifacts and Markdown uses private `deep-research://` links. `GET /sessions/{id}/artifacts/{name}` checks session ownership before serving an image; an unavailable provider yields a warning but never fails the text report. Runners live in `packages/harness/deerflow/agents/deep_research/runners/`, may never import `app.*`, and use only the `AdapterBundle`. |
|
||||
| **Gateway auth** | Owner-scoped Deep Research routes await the platform's asynchronous `get_current_user` resolver before they read or write persistence, ensuring each session and job operation receives a concrete user ID. Deep Research artifacts are always resolved from a whitelisted filename via `/mnt/user-data/outputs/<name>` before mapping to a host path, preserving the sandbox guard on native Windows and Linux hosts. |
|
||||
@@ -1527,8 +1527,10 @@ same DOM value, so a fetched-and-revoked object URL is not acceptable.
|
||||
Third-party pages embed published Wiki content through the chrome-free route
|
||||
`/#/embed/knowledge/{mapping_id}` (optional exact-page `wikiSlug`). It defaults
|
||||
to a recognized Wiki index page (title `索引`/`index`, page type `index`, or an
|
||||
`index`/`home`/`readme` slug), promotes that page to the first public-directory
|
||||
entry, then falls back to an overview/root page when no index exists. It retains
|
||||
`index`/`home`/`readme` slug) and promotes that page to the first public-directory
|
||||
entry. When upstream pages exist without an explicit index shell, the frontend
|
||||
injects a virtual `__index__` page and builds its catalogue from the page list;
|
||||
an all-empty dynamic index response must also fall back to that page list. It retains
|
||||
directory navigation and supports the shared public-embed `theme` / `bg` /
|
||||
`text` appearance inputs. WeKnora dynamically renders index page-type counts,
|
||||
nested category headings, page summaries, and `data-slug` links even though the
|
||||
@@ -1544,9 +1546,14 @@ the loaded page title, slug, or alias and rewrites them to exact embedded
|
||||
`wikiSlug` routes. Preserve WeKnora's `wiki-content-link` interaction: primary
|
||||
brand color, medium weight, dashed underline becoming solid on hover, focus
|
||||
outline, and in-reader SPA navigation rather than a page reload. `GET
|
||||
/api/public/llmwiki/knowledge-bases` is the
|
||||
anonymous discovery endpoint and returns only each published ordinary mapping's
|
||||
`id` and `name` (plus the collection total). All anonymous reader calls use that
|
||||
/api/public/llmwiki/knowledge-bases` is the anonymous discovery endpoint. It
|
||||
validates published ordinary mappings against WeKnora, drops stale mappings,
|
||||
and returns safe comprehensive metadata from the WeKnora list/detail/page APIs:
|
||||
document/chunk/processing/share counts, non-secret configuration and timestamps,
|
||||
Wiki page type/status counts and explicit/generated index entry, plus aggregate
|
||||
local-vector state/counts/timestamps. Never expose remote WeKnora ids,
|
||||
tenant/creator identity, storage credentials, API credentials, raw vectors, or
|
||||
per-page indexing errors. All anonymous reader calls use that
|
||||
public namespace and the `/knowledge-bases/{id}` Wiki-page children. Those
|
||||
endpoints must resolve with a synthetic anonymous user and `is_admin=False`,
|
||||
explicitly require `publication_status=published`, and hard-deny the system
|
||||
|
||||
@@ -153,6 +153,7 @@ FastAPI application providing REST endpoints for frontend integration:
|
||||
| `POST /api/threads/{id}/uploads` | Upload files (auto-converts PDF/PPT/Excel/Word to Markdown, rejects directory paths) |
|
||||
| `GET /api/threads/{id}/uploads/list` | List uploaded files |
|
||||
| `DELETE /api/threads/{id}` | Delete DeerFlow-managed local thread data after LangGraph thread deletion; unexpected failures are logged server-side and return a generic 500 detail |
|
||||
| `GET /api/artifact-library` | List the current user's generated conversation files across normal threads, with filename/conversation search, file-kind filtering, pagination, and an optional cache-bypassing refresh. The filesystem `outputs/` directories remain authoritative; uploads and image/audio/video files are excluded. |
|
||||
| `GET /api/threads/{id}/artifacts/{path}` | Serve generated artifacts |
|
||||
| `GET/POST /api/deep-research/*` | Durable Deep Research sessions and jobs. `quick`/`basic` run the compact report flow; `detailed` plans subtopics and writes independent sections; `deep` recursively investigates bounded follow-up branches. All modes reuse DeerFlow's configured material provider, LLM factory, hidden-thread artifacts, and SSE event log. Deep Research session/job/event/source/message writes serialise their rows in the writer transaction, rather than doing a post-commit ORM refresh or readback; this prevents MySQL read/write-splitting replica lag from failing “start writing”, dispatcher claim, report persistence, or follow-up history. Whole-document artifact/report rewrites likewise create their reversible version snapshots from writer-local ORM data and never refresh after commit, so replica lag cannot restore an otherwise successful rewrite. A report is written to the session and confirmed before its Job becomes `completed`; an unconfirmed write is retried and never exposed as a false completion. Retrying an already-created queued job re-nudges the dispatcher, recovering an interrupted HTTP response without creating a second report. If progress-event persistence, durable replay, or live fan-out is briefly unavailable, the worker/stream logs the degradation and continues or reconnects rather than failing the whole job. Whole-report rewrite accepts an optional `reportOutline` Markdown template; the durable job persists it and applies it as the chapter structure constraint. Optional report illustrations use a separately configured OpenAI-compatible image endpoint and are served only through the owning session's authenticated artifact route. |
|
||||
| `/api/enterprise-research/*` | Temporarily disabled. Its router and persistence initialization are not registered, so these endpoints return 404 and no enterprise-research tables are created or migrated during startup. |
|
||||
@@ -865,8 +866,10 @@ report-structure column for databases stamped by either old variant.
|
||||
Published knowledge bases also have a login-free, chrome-free reader at
|
||||
`/#/embed/knowledge/{mapping_id}` for direct third-party iframe use. With no
|
||||
`wikiSlug`, it selects the Wiki **索引** page first (recognized by title, page
|
||||
type, or `index`/`home`/`readme` slug), pins that page to the top of the public
|
||||
directory, and otherwise falls back to an overview/root page;
|
||||
type, or `index`/`home`/`readme` slug) and pins that page to the top of the
|
||||
public directory. If WeKnora has pages but no explicit index page, the reader
|
||||
creates a virtual `__index__` landing page from the public directory; empty
|
||||
dynamic-index groups also fall back to those page summaries;
|
||||
`?wikiSlug=...` deep-links one page. The reader supports
|
||||
`theme=light|dark`, `bg`, and `text` embed appearance parameters. Index-article
|
||||
catalogue sections (page-type counts, nested categories, page summaries) are
|
||||
@@ -882,8 +885,14 @@ only from the read-only `/api/public/llmwiki/knowledge-bases/{id}/...` namespace
|
||||
which rechecks `publication_status=published` on every request and denies the
|
||||
system conversation-deposit knowledge base. Private or unpublished bases are
|
||||
never exposed by this route. Anonymous consumers can discover the available
|
||||
public Wiki libraries through `GET /api/public/llmwiki/knowledge-bases`; each
|
||||
entry deliberately contains only the DeerFlow mapping `id` and display `name`.
|
||||
public Wiki libraries through `GET /api/public/llmwiki/knowledge-bases`. The
|
||||
endpoint validates each published mapping against WeKnora, omits stale remote
|
||||
mappings, and returns safe comprehensive metadata: description/type,
|
||||
document/chunk/processing/share counts, non-secret configuration and
|
||||
timestamps, Wiki page/type/status counts plus explicit/generated index mode and
|
||||
default embed route, and aggregate local-vector state/counts/timestamps. It
|
||||
never returns the WeKnora id, tenant/creator identity, storage credentials, API
|
||||
credentials, raw vectors, or per-page indexing errors.
|
||||
Protected `resource://` and supported object-storage images are fetched through
|
||||
the current mapping instead of being exposed as upstream URLs. Authenticated
|
||||
readers use `GET /api/llmwiki/knowledge-bases/{id}/files?file_path=...`; public
|
||||
|
||||
@@ -1,11 +1,15 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import mimetypes
|
||||
import time
|
||||
import zipfile
|
||||
from datetime import datetime, timezone
|
||||
from collections import OrderedDict
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
from urllib.parse import quote
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from fastapi import APIRouter, HTTPException, Query, Request
|
||||
from fastapi.responses import FileResponse, PlainTextResponse, Response
|
||||
from pydantic import BaseModel
|
||||
|
||||
@@ -13,6 +17,7 @@ from app.gateway.authz import require_permission
|
||||
from app.gateway.path_utils import aresolve_thread_virtual_path
|
||||
from deerflow.config.paths import get_paths
|
||||
from deerflow.runtime.thread_paths import aresolve_path_user_id
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -38,6 +43,84 @@ EDITABLE_TEXT_SUFFIXES = {
|
||||
|
||||
_VIRTUAL_USER_DATA_PREFIX = "mnt/user-data"
|
||||
|
||||
ArtifactLibraryKind = Literal["all", "document", "spreadsheet", "text_data", "archive", "other"]
|
||||
|
||||
_MEDIA_MIME_PREFIXES = ("image/", "audio/", "video/")
|
||||
_MEDIA_SUFFIXES = {
|
||||
".aac",
|
||||
".aiff",
|
||||
".ape",
|
||||
".avi",
|
||||
".bmp",
|
||||
".flac",
|
||||
".gif",
|
||||
".heic",
|
||||
".ico",
|
||||
".jpeg",
|
||||
".jpg",
|
||||
".m4a",
|
||||
".m4v",
|
||||
".mkv",
|
||||
".mov",
|
||||
".mp3",
|
||||
".mp4",
|
||||
".ogg",
|
||||
".png",
|
||||
".svg",
|
||||
".tiff",
|
||||
".wav",
|
||||
".webm",
|
||||
".webp",
|
||||
".wma",
|
||||
}
|
||||
_DOCUMENT_SUFFIXES = {
|
||||
".doc",
|
||||
".docx",
|
||||
".epub",
|
||||
".odt",
|
||||
".pdf",
|
||||
".ppt",
|
||||
".pptx",
|
||||
".rtf",
|
||||
}
|
||||
_SPREADSHEET_SUFFIXES = {".csv", ".ods", ".tsv", ".xls", ".xlsx"}
|
||||
_TEXT_DATA_SUFFIXES = {
|
||||
".bash",
|
||||
".c",
|
||||
".conf",
|
||||
".cpp",
|
||||
".css",
|
||||
".go",
|
||||
".h",
|
||||
".html",
|
||||
".ini",
|
||||
".java",
|
||||
".js",
|
||||
".json",
|
||||
".jsonl",
|
||||
".log",
|
||||
".markdown",
|
||||
".md",
|
||||
".mmd",
|
||||
".py",
|
||||
".rs",
|
||||
".sh",
|
||||
".sql",
|
||||
".toml",
|
||||
".ts",
|
||||
".tsx",
|
||||
".txt",
|
||||
".xml",
|
||||
".yaml",
|
||||
".yml",
|
||||
}
|
||||
_ARCHIVE_SUFFIXES = {".7z", ".gz", ".rar", ".skill", ".tar", ".tgz", ".zip"}
|
||||
_ARTIFACT_LIBRARY_CACHE_TTL_SECONDS = 15.0
|
||||
_ARTIFACT_LIBRARY_CACHE_MAX_USERS = 128
|
||||
_ARTIFACT_LIBRARY_SCAN_CONCURRENCY = 8
|
||||
_ARTIFACT_LIBRARY_THREAD_BATCH_SIZE = 500
|
||||
_artifact_library_cache: OrderedDict[str, tuple[float, list[dict]]] = OrderedDict()
|
||||
|
||||
|
||||
def coerce_artifact_virtual_path(path: str) -> str:
|
||||
"""Normalize an artifacts URL path to a sandbox virtual path.
|
||||
@@ -60,6 +143,26 @@ def coerce_artifact_virtual_path(path: str) -> str:
|
||||
|
||||
class ArtifactUpdateRequest(BaseModel):
|
||||
content: str
|
||||
|
||||
|
||||
class ArtifactLibraryItem(BaseModel):
|
||||
thread_id: str
|
||||
thread_title: str
|
||||
agent_id: str | None = None
|
||||
thread_type: str | None = None
|
||||
path: str
|
||||
name: str
|
||||
size_bytes: int
|
||||
mime_type: str | None = None
|
||||
modified_at: str
|
||||
file_kind: str
|
||||
|
||||
|
||||
class ArtifactLibraryResponse(BaseModel):
|
||||
total: int
|
||||
offset: int
|
||||
limit: int
|
||||
items: list[ArtifactLibraryItem]
|
||||
|
||||
|
||||
def _build_content_disposition(disposition_type: str, filename: str) -> str:
|
||||
@@ -139,21 +242,184 @@ def collect_outputs_listing(outputs_dir: Path) -> list[dict]:
|
||||
resolved.relative_to(outputs_root)
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
relative = resolved.relative_to(outputs_root).as_posix()
|
||||
mime_type, _ = mimetypes.guess_type(resolved.name)
|
||||
stat = resolved.stat()
|
||||
try:
|
||||
relative = resolved.relative_to(outputs_root).as_posix()
|
||||
mime_type, _ = mimetypes.guess_type(resolved.name)
|
||||
stat = resolved.stat()
|
||||
except OSError:
|
||||
# A run may atomically replace or remove a file while this directory
|
||||
# is being scanned. Treat that transient entry as absent instead of
|
||||
# failing the complete thread/library listing.
|
||||
continue
|
||||
files.append(
|
||||
{
|
||||
"path": f"mnt/user-data/outputs/{relative}",
|
||||
"name": resolved.name,
|
||||
"size_bytes": stat.st_size,
|
||||
"mime_type": mime_type,
|
||||
"modified_at": datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc).isoformat(),
|
||||
"modified_at": datetime.fromtimestamp(stat.st_mtime, tz=UTC).isoformat(),
|
||||
}
|
||||
)
|
||||
|
||||
files.sort(key=lambda f: f["modified_at"], reverse=True)
|
||||
return files
|
||||
|
||||
|
||||
def artifact_library_kind(name: str) -> str:
|
||||
"""Classify one non-media artifact for the library filters."""
|
||||
suffix = Path(name).suffix.lower()
|
||||
if suffix in _DOCUMENT_SUFFIXES:
|
||||
return "document"
|
||||
if suffix in _SPREADSHEET_SUFFIXES:
|
||||
return "spreadsheet"
|
||||
if suffix in _TEXT_DATA_SUFFIXES:
|
||||
return "text_data"
|
||||
if suffix in _ARCHIVE_SUFFIXES:
|
||||
return "archive"
|
||||
return "other"
|
||||
|
||||
|
||||
def is_artifact_library_file(file: dict) -> bool:
|
||||
"""Return whether a generated file belongs in the document-only library."""
|
||||
mime_type = str(file.get("mime_type") or "").lower()
|
||||
if mime_type.startswith(_MEDIA_MIME_PREFIXES):
|
||||
return False
|
||||
return Path(str(file.get("name") or file.get("path") or "")).suffix.lower() not in _MEDIA_SUFFIXES
|
||||
|
||||
|
||||
async def _search_artifact_library_threads(thread_store) -> list[dict]:
|
||||
"""Load every normal conversation owned by the current request user."""
|
||||
rows: list[dict] = []
|
||||
seen: set[str] = set()
|
||||
offset = 0
|
||||
while True:
|
||||
batch = await thread_store.search(
|
||||
limit=_ARTIFACT_LIBRARY_THREAD_BATCH_SIZE,
|
||||
offset=offset,
|
||||
exclude_system=True,
|
||||
)
|
||||
if not batch:
|
||||
break
|
||||
added = 0
|
||||
for row in batch:
|
||||
thread_id = str(row.get("thread_id") or "")
|
||||
if not thread_id or thread_id in seen:
|
||||
continue
|
||||
seen.add(thread_id)
|
||||
rows.append(row)
|
||||
added += 1
|
||||
if len(batch) < _ARTIFACT_LIBRARY_THREAD_BATCH_SIZE or added == 0:
|
||||
break
|
||||
offset += len(batch)
|
||||
return rows
|
||||
|
||||
|
||||
async def _scan_artifact_library(thread_store) -> list[dict]:
|
||||
"""Scan generated outputs for all current-user conversation threads."""
|
||||
rows = await _search_artifact_library_threads(thread_store)
|
||||
paths = get_paths()
|
||||
semaphore = asyncio.Semaphore(_ARTIFACT_LIBRARY_SCAN_CONCURRENCY)
|
||||
|
||||
async def scan_thread(row: dict) -> list[dict]:
|
||||
thread_id = str(row.get("thread_id") or "")
|
||||
if not thread_id:
|
||||
return []
|
||||
async with semaphore:
|
||||
try:
|
||||
path_user_id = await aresolve_path_user_id(thread_id)
|
||||
outputs_dir = paths.sandbox_outputs_dir(thread_id, user_id=path_user_id)
|
||||
files = await asyncio.to_thread(collect_outputs_listing, outputs_dir)
|
||||
except Exception:
|
||||
logger.warning("Failed to scan artifacts for library thread %s", thread_id, exc_info=True)
|
||||
return []
|
||||
|
||||
metadata = row.get("metadata") if isinstance(row.get("metadata"), dict) else {}
|
||||
agent_id = metadata.get("agent_id")
|
||||
thread_type = metadata.get("thread_type")
|
||||
title = str(row.get("display_name") or "新对话")
|
||||
items: list[dict] = []
|
||||
for file in files:
|
||||
if not is_artifact_library_file(file):
|
||||
continue
|
||||
items.append(
|
||||
{
|
||||
"thread_id": thread_id,
|
||||
"thread_title": title,
|
||||
"agent_id": agent_id if isinstance(agent_id, str) and agent_id else None,
|
||||
"thread_type": thread_type if isinstance(thread_type, str) and thread_type else None,
|
||||
**file,
|
||||
"file_kind": artifact_library_kind(str(file.get("name") or "")),
|
||||
}
|
||||
)
|
||||
return items
|
||||
|
||||
nested = await asyncio.gather(*(scan_thread(row) for row in rows))
|
||||
items = [item for group in nested for item in group]
|
||||
items.sort(
|
||||
key=lambda item: (
|
||||
str(item.get("modified_at") or ""),
|
||||
str(item.get("thread_id") or ""),
|
||||
str(item.get("path") or ""),
|
||||
),
|
||||
reverse=True,
|
||||
)
|
||||
return items
|
||||
|
||||
|
||||
async def _cached_artifact_library(thread_store, *, refresh: bool) -> list[dict]:
|
||||
user_id = str(get_effective_user_id() or "default")
|
||||
now = time.monotonic()
|
||||
cached = _artifact_library_cache.get(user_id)
|
||||
if not refresh and cached and cached[0] > now:
|
||||
_artifact_library_cache.move_to_end(user_id)
|
||||
return cached[1]
|
||||
|
||||
items = await _scan_artifact_library(thread_store)
|
||||
_artifact_library_cache[user_id] = (now + _ARTIFACT_LIBRARY_CACHE_TTL_SECONDS, items)
|
||||
_artifact_library_cache.move_to_end(user_id)
|
||||
while len(_artifact_library_cache) > _ARTIFACT_LIBRARY_CACHE_MAX_USERS:
|
||||
_artifact_library_cache.popitem(last=False)
|
||||
return items
|
||||
|
||||
|
||||
@router.get(
|
||||
"/artifact-library",
|
||||
response_model=ArtifactLibraryResponse,
|
||||
summary="List Generated Files Across Conversations",
|
||||
description="Aggregate non-media files generated in the current user's normal conversations.",
|
||||
)
|
||||
@require_permission("threads", "read")
|
||||
async def list_artifact_library(
|
||||
request: Request,
|
||||
query: str | None = Query(default=None, max_length=200),
|
||||
file_kind: ArtifactLibraryKind = Query(default="all"),
|
||||
offset: int = Query(default=0, ge=0),
|
||||
limit: int = Query(default=50, ge=1, le=200),
|
||||
refresh: bool = Query(default=False),
|
||||
) -> ArtifactLibraryResponse:
|
||||
"""Return a flat, newest-first library of generated conversation files.
|
||||
|
||||
The filesystem remains the source of truth: every owned, non-system thread's
|
||||
``outputs`` directory is scanned, including files written without an
|
||||
explicit ``present_files`` call. Uploads and image/audio/video outputs are
|
||||
intentionally outside this collection.
|
||||
"""
|
||||
from app.gateway.deps import get_thread_store
|
||||
|
||||
items = await _cached_artifact_library(get_thread_store(request), refresh=refresh)
|
||||
normalized_query = (query or "").strip().casefold()
|
||||
filtered = [
|
||||
item
|
||||
for item in items
|
||||
if (file_kind == "all" or item["file_kind"] == file_kind) and (not normalized_query or normalized_query in str(item.get("name") or "").casefold() or normalized_query in str(item.get("thread_title") or "").casefold())
|
||||
]
|
||||
page = filtered[offset : offset + limit]
|
||||
return ArtifactLibraryResponse(
|
||||
total=len(filtered),
|
||||
offset=offset,
|
||||
limit=limit,
|
||||
items=[ArtifactLibraryItem.model_validate(item) for item in page],
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
|
||||
@@ -55,6 +55,7 @@ from deerflow.config.agents_config import load_agent_config
|
||||
from deerflow.config.llmwiki_config import LlmWikiRuntimeOverride, normalize_service_url
|
||||
from deerflow.config.system_settings import load_system_settings, save_system_settings
|
||||
from deerflow.integrations.weknora.client import WeKnoraClient, WeKnoraError
|
||||
from deerflow.integrations.weknora.local_index.retrieval import is_fully_vectorized
|
||||
from deerflow.integrations.weknora.local_index.search import WikiIndexError
|
||||
from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
|
||||
from deerflow.persistence.llmwiki import LlmWikiStore
|
||||
@@ -496,15 +497,7 @@ def _should_follow_weknora_binary_redirect(path: str) -> bool:
|
||||
"""Keep protected preview/image bytes inside the DeerFlow proxy."""
|
||||
|
||||
segments = _path_segments(path)
|
||||
return (
|
||||
len(segments) == 5
|
||||
and segments[:3] == ["api", "v1", "knowledge"]
|
||||
and segments[4] == "preview"
|
||||
) or (
|
||||
len(segments) == 5
|
||||
and segments[:3] == ["api", "v1", "knowledge-bases"]
|
||||
and segments[4] == "files"
|
||||
)
|
||||
return (len(segments) == 5 and segments[:3] == ["api", "v1", "knowledge"] and segments[4] == "preview") or (len(segments) == 5 and segments[:3] == ["api", "v1", "knowledge-bases"] and segments[4] == "files")
|
||||
|
||||
|
||||
def _forward_request_headers(request: Request) -> dict[str, str]:
|
||||
@@ -1833,6 +1826,9 @@ def _mapping_view(row: dict[str, Any], remote: dict[str, Any] | None, *, actor_u
|
||||
"knowledge_count": int(source.get("knowledge_count") or 0),
|
||||
"chunk_count": int(source.get("chunk_count") or 0),
|
||||
"processing_count": int(source.get("processing_count") or 0),
|
||||
"share_count": int(source.get("share_count") or 0),
|
||||
"is_processing": bool(source.get("is_processing") or int(source.get("processing_count") or 0) > 0),
|
||||
"is_temporary": bool(source.get("is_temporary", False)),
|
||||
"embedding_model_id": source.get("embedding_model_id"),
|
||||
"vector_store_id": source.get("vector_store_id"),
|
||||
"vector_store_name": source.get("vector_store_name"),
|
||||
@@ -1840,16 +1836,23 @@ def _mapping_view(row: dict[str, Any], remote: dict[str, Any] | None, *, actor_u
|
||||
"vector_store_engine_type": source.get("vector_store_engine_type"),
|
||||
"vector_store_status": source.get("vector_store_status"),
|
||||
"chunking_config": source.get("chunking_config") if isinstance(source.get("chunking_config"), dict) else {},
|
||||
"image_processing_config": (source.get("image_processing_config") if isinstance(source.get("image_processing_config"), dict) else {}),
|
||||
"indexing_strategy": source.get("indexing_strategy") if isinstance(source.get("indexing_strategy"), dict) else {},
|
||||
"capabilities": source.get("capabilities") if isinstance(source.get("capabilities"), dict) else {},
|
||||
"extract_config": source.get("extract_config") if isinstance(source.get("extract_config"), dict) else {},
|
||||
"faq_config": source.get("faq_config") if isinstance(source.get("faq_config"), dict) else {},
|
||||
"wiki_config": source.get("wiki_config") if isinstance(source.get("wiki_config"), dict) else {},
|
||||
"question_generation_config": (source.get("question_generation_config") if isinstance(source.get("question_generation_config"), dict) else {}),
|
||||
"auto_tag_config": source.get("auto_tag_config") if isinstance(source.get("auto_tag_config"), dict) else {},
|
||||
"vlm_config": source.get("vlm_config") if isinstance(source.get("vlm_config"), dict) else {},
|
||||
"asr_config": source.get("asr_config") if isinstance(source.get("asr_config"), dict) else {},
|
||||
"summary_model_id": source.get("summary_model_id"),
|
||||
"storage_provider": (source.get("storage_provider_config", {}).get("provider") if isinstance(source.get("storage_provider_config"), dict) else None),
|
||||
"created_at": row.get("created_at"),
|
||||
"updated_at": row.get("updated_at"),
|
||||
"published_at": row.get("published_at"),
|
||||
"remote_created_at": source.get("created_at"),
|
||||
"remote_updated_at": source.get("updated_at"),
|
||||
}
|
||||
|
||||
|
||||
@@ -1877,11 +1880,7 @@ async def _public_mapping(request: Request, mapping_id: str) -> dict[str, Any]:
|
||||
write=False,
|
||||
is_admin=False,
|
||||
)
|
||||
if (
|
||||
row is None
|
||||
or row.get("publication_status") != "published"
|
||||
or _is_conversation_deposit_mapping(row)
|
||||
):
|
||||
if row is None or row.get("publication_status") != "published" or _is_conversation_deposit_mapping(row):
|
||||
raise HTTPException(status_code=404, detail="Published knowledge base not found")
|
||||
return row
|
||||
|
||||
@@ -2537,24 +2536,260 @@ async def test_llmwiki_connection(request: Request, body: LlmWikiConnectionTest)
|
||||
return {"success": True, "provider": "weknora", "health": health}
|
||||
|
||||
|
||||
_PUBLIC_WIKI_SUMMARY_CONCURRENCY = 8
|
||||
_PUBLIC_WIKI_PAGE_SIZE = 500
|
||||
_PUBLIC_WIKI_PAGE_LIMIT = 100
|
||||
_GENERATED_PUBLIC_WIKI_INDEX_SLUG = "__index__"
|
||||
|
||||
|
||||
def _is_public_wiki_index_page(page: dict[str, Any]) -> bool:
|
||||
page_type = str(page.get("page_type") or "").strip().lower()
|
||||
title = str(page.get("title") or "").strip().lower()
|
||||
slug_name = str(page.get("slug") or "").strip("/").rsplit("/", 1)[-1].lower()
|
||||
return page_type == "index" or title in {"索引", "index"} or slug_name in {"index", "home", "readme"}
|
||||
|
||||
|
||||
def _public_wiki_default_page(pages: list[dict[str, Any]]) -> dict[str, Any] | None:
|
||||
return (
|
||||
next((page for page in pages if _is_public_wiki_index_page(page)), None)
|
||||
or next((page for page in pages if str(page.get("page_type") or "").lower() == "overview"), None)
|
||||
or next(
|
||||
(page for page in pages if not str(page.get("parent_slug") or "").strip() and int(page.get("depth") or 0) == 0),
|
||||
None,
|
||||
)
|
||||
or (pages[0] if pages else None)
|
||||
)
|
||||
|
||||
|
||||
def _count_public_wiki_values(pages: list[dict[str, Any]], key: str, fallback: str) -> dict[str, int]:
|
||||
counts: dict[str, int] = {}
|
||||
for page in pages:
|
||||
value = str(page.get(key) or fallback).strip().lower() or fallback
|
||||
counts[value] = counts.get(value, 0) + 1
|
||||
return counts
|
||||
|
||||
|
||||
def _public_wiki_time_bound(pages: list[dict[str, Any]], key: str, *, latest: bool) -> str | None:
|
||||
values = [str(page.get(key) or "").strip() for page in pages]
|
||||
values = [value for value in values if value]
|
||||
if not values:
|
||||
return None
|
||||
return max(values) if latest else min(values)
|
||||
|
||||
|
||||
async def _all_public_wiki_page_views(
|
||||
client: WeKnoraClient,
|
||||
remote_id: str,
|
||||
) -> tuple[list[dict[str, Any]], int, bool]:
|
||||
pages_by_slug: dict[str, dict[str, Any]] = {}
|
||||
remote_total = 0
|
||||
complete = False
|
||||
for page_number in range(1, _PUBLIC_WIKI_PAGE_LIMIT + 1):
|
||||
result = await client.list_wiki_pages(
|
||||
remote_id,
|
||||
page=page_number,
|
||||
page_size=_PUBLIC_WIKI_PAGE_SIZE,
|
||||
query="",
|
||||
)
|
||||
raw_pages = result.get("pages") if isinstance(result, dict) else []
|
||||
page_items = raw_pages if isinstance(raw_pages, list) else []
|
||||
for page_item in page_items:
|
||||
if not isinstance(page_item, dict):
|
||||
continue
|
||||
page = _wiki_page_view(page_item)
|
||||
if page is None:
|
||||
continue
|
||||
slug = str(page.get("slug") or "").strip()
|
||||
pages_by_slug[slug or f"__page_{len(pages_by_slug)}"] = page
|
||||
reported_total = int(result.get("total") or 0)
|
||||
remote_total = max(remote_total, reported_total, len(pages_by_slug))
|
||||
total_pages = max(1, int(result.get("total_pages") or 1))
|
||||
if not page_items or page_number >= total_pages or (reported_total > 0 and len(pages_by_slug) >= remote_total):
|
||||
complete = True
|
||||
break
|
||||
return list(pages_by_slug.values()), max(remote_total, len(pages_by_slug)), complete
|
||||
|
||||
|
||||
def _public_wiki_summary(
|
||||
mapping_id: str,
|
||||
pages: list[dict[str, Any]],
|
||||
*,
|
||||
total: int,
|
||||
complete: bool,
|
||||
) -> dict[str, Any]:
|
||||
explicit_index = next((page for page in pages if _is_public_wiki_index_page(page)), None)
|
||||
default_page = _public_wiki_default_page(pages)
|
||||
if explicit_index is not None:
|
||||
index_mode = "explicit"
|
||||
index_slug: str | None = str(explicit_index.get("slug") or "") or None
|
||||
elif pages:
|
||||
index_mode = "generated"
|
||||
index_slug = _GENERATED_PUBLIC_WIKI_INDEX_SLUG
|
||||
else:
|
||||
index_mode = "none"
|
||||
index_slug = None
|
||||
default_slug = index_slug or (str(default_page.get("slug") or "") if default_page else None)
|
||||
route = f"/embed/knowledge/{quote(mapping_id, safe='')}"
|
||||
if default_slug:
|
||||
route = f"{route}?{urlencode({'wikiSlug': default_slug})}"
|
||||
return {
|
||||
"status": "ready" if pages else "empty",
|
||||
"page_count": total,
|
||||
"loaded_page_count": len(pages),
|
||||
"stats_complete": complete,
|
||||
"page_type_counts": _count_public_wiki_values(pages, "page_type", "page"),
|
||||
"page_status_counts": _count_public_wiki_values(pages, "status", "unknown"),
|
||||
"index_mode": index_mode,
|
||||
"has_explicit_index": explicit_index is not None,
|
||||
"index_slug": index_slug,
|
||||
"default_slug": default_slug,
|
||||
"created_at": _public_wiki_time_bound(pages, "created_at", latest=False),
|
||||
"updated_at": _public_wiki_time_bound(pages, "updated_at", latest=True),
|
||||
"embed_route": route,
|
||||
}
|
||||
|
||||
|
||||
def _unavailable_public_wiki_summary(mapping_id: str) -> dict[str, Any]:
|
||||
return {
|
||||
"status": "unavailable",
|
||||
"page_count": 0,
|
||||
"loaded_page_count": 0,
|
||||
"stats_complete": False,
|
||||
"page_type_counts": {},
|
||||
"page_status_counts": {},
|
||||
"index_mode": "unknown",
|
||||
"has_explicit_index": False,
|
||||
"index_slug": None,
|
||||
"default_slug": None,
|
||||
"created_at": None,
|
||||
"updated_at": None,
|
||||
"embed_route": f"/embed/knowledge/{quote(mapping_id, safe='')}",
|
||||
}
|
||||
|
||||
|
||||
def _public_local_index_summary(
|
||||
request: Request,
|
||||
mapping: dict[str, Any],
|
||||
state: dict[str, Any] | None,
|
||||
) -> dict[str, Any]:
|
||||
app_state = request.app.state
|
||||
local_config = getattr(
|
||||
getattr(getattr(app_state, "config", None), "llmwiki", None),
|
||||
"local_wiki_index",
|
||||
None,
|
||||
)
|
||||
enabled = bool(getattr(local_config, "enabled", False))
|
||||
current = dict(state or {})
|
||||
state_name = str(current.get("state") or "not_synced")
|
||||
if state_name == "idle" and int(current.get("index_revision") or 0) == 0 and not current.get("last_completed_at") and not current.get("last_success_at"):
|
||||
state_name = "not_synced"
|
||||
embedding = getattr(app_state, "llmwiki_embedding", None)
|
||||
fingerprint = embedding.fingerprint if embedding is not None else None
|
||||
fully_vectorized = bool(mapping.get("wiki_index_enabled", True)) and is_fully_vectorized(
|
||||
current,
|
||||
fingerprint,
|
||||
)
|
||||
return {
|
||||
"enabled": enabled,
|
||||
"wiki_index_enabled": bool(mapping.get("wiki_index_enabled", True)),
|
||||
"state": state_name,
|
||||
"fully_vectorized": fully_vectorized,
|
||||
"retrieval_mode": "vector" if fully_vectorized else "wiki_api",
|
||||
"remote_page_count": int(current.get("remote_page_count") or 0),
|
||||
"local_page_count": int(current.get("local_page_count") or 0),
|
||||
"ready_page_count": int(current.get("ready_page_count") or 0),
|
||||
"failed_page_count": int(current.get("failed_page_count") or 0),
|
||||
"vector_count": int(current.get("vector_count") or 0),
|
||||
"last_started_at": current.get("last_started_at"),
|
||||
"last_completed_at": current.get("last_completed_at"),
|
||||
"last_success_at": current.get("last_success_at"),
|
||||
"updated_at": current.get("updated_at"),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/api/public/llmwiki/knowledge-bases")
|
||||
async def list_public_knowledge_bases(request: Request) -> dict[str, Any]:
|
||||
"""List the minimal identity of every knowledge base published as a public Wiki."""
|
||||
"""List live, published Wiki libraries with safe metadata and index health."""
|
||||
|
||||
rows = await _store(request).list_visible(
|
||||
"__public_wiki__",
|
||||
scope="public",
|
||||
is_admin=False,
|
||||
)
|
||||
knowledge_bases = [
|
||||
{
|
||||
"id": str(row["id"]),
|
||||
"name": str(row.get("name") or ""),
|
||||
}
|
||||
for row in rows
|
||||
if row.get("publication_status") == "published"
|
||||
and not _is_conversation_deposit_mapping(row)
|
||||
rows = [
|
||||
row
|
||||
for row in await _store(request).list_visible(
|
||||
"__public_wiki__",
|
||||
scope="public",
|
||||
is_admin=False,
|
||||
)
|
||||
if row.get("publication_status") == "published" and not _is_conversation_deposit_mapping(row)
|
||||
]
|
||||
client = _client_or_503()
|
||||
try:
|
||||
remote_items = await client.list_knowledge_bases()
|
||||
except WeKnoraError as exc:
|
||||
_raise_upstream(exc)
|
||||
remote_by_id = {str(item.get("id")): item for item in remote_items if isinstance(item, dict) and item.get("id")}
|
||||
# A successful upstream list is authoritative: stale DeerFlow mappings are
|
||||
# omitted so discovery never advertises a Wiki that its detail route cannot open.
|
||||
rows = [row for row in rows if str(row.get("weknora_id") or "") in remote_by_id]
|
||||
mapping_ids = [str(row["id"]) for row in rows]
|
||||
index_states: dict[str, dict[str, Any]] = {}
|
||||
index_store = getattr(request.app.state, "llmwiki_index_store", None)
|
||||
if index_store is not None and mapping_ids:
|
||||
try:
|
||||
index_states = {str(state.get("knowledge_base_mapping_id") or ""): state for state in await index_store.list_index_status(mapping_ids) if isinstance(state, dict)}
|
||||
except Exception:
|
||||
logger.warning("Failed to read public Wiki local-index summaries", exc_info=True)
|
||||
|
||||
semaphore = asyncio.Semaphore(_PUBLIC_WIKI_SUMMARY_CONCURRENCY)
|
||||
|
||||
async def build_summary(row: dict[str, Any]) -> dict[str, Any]:
|
||||
mapping_id = str(row["id"])
|
||||
remote_id = str(row["weknora_id"])
|
||||
remote = remote_by_id[remote_id]
|
||||
async with semaphore:
|
||||
try:
|
||||
remote = {**remote, **await client.get_knowledge_base(remote_id)}
|
||||
except WeKnoraError as exc:
|
||||
if exc.status_code == 404:
|
||||
return {}
|
||||
logger.warning(
|
||||
"Failed to load public Wiki detail metadata: mapping_id=%s",
|
||||
mapping_id,
|
||||
exc_info=True,
|
||||
)
|
||||
try:
|
||||
pages, page_total, complete = await _all_public_wiki_page_views(client, remote_id)
|
||||
wiki = _public_wiki_summary(
|
||||
mapping_id,
|
||||
pages,
|
||||
total=page_total,
|
||||
complete=complete,
|
||||
)
|
||||
except WeKnoraError as exc:
|
||||
logger.warning(
|
||||
"Failed to load public Wiki page summary: mapping_id=%s",
|
||||
mapping_id,
|
||||
exc_info=True,
|
||||
)
|
||||
wiki = (
|
||||
_public_wiki_summary(mapping_id, [], total=0, complete=True)
|
||||
if exc.status_code == 404
|
||||
else _unavailable_public_wiki_summary(mapping_id)
|
||||
)
|
||||
return {
|
||||
**_mapping_view(
|
||||
row,
|
||||
remote,
|
||||
actor_user_id="__public_wiki__",
|
||||
is_admin=False,
|
||||
),
|
||||
"wiki": wiki,
|
||||
"local_index": _public_local_index_summary(
|
||||
request,
|
||||
row,
|
||||
index_states.get(mapping_id),
|
||||
),
|
||||
}
|
||||
|
||||
knowledge_bases = [summary for summary in await asyncio.gather(*(build_summary(row) for row in rows)) if summary]
|
||||
return {"knowledge_bases": knowledge_bases, "total": len(knowledge_bases)}
|
||||
|
||||
|
||||
@@ -2596,13 +2831,7 @@ async def list_public_wiki_pages(
|
||||
except WeKnoraError as exc:
|
||||
_raise_upstream(exc)
|
||||
raw_pages = result.get("pages") if isinstance(result, dict) else []
|
||||
pages = [
|
||||
view
|
||||
for page_item in (raw_pages if isinstance(raw_pages, list) else [])
|
||||
if isinstance(page_item, dict)
|
||||
for view in [_wiki_page_view(page_item)]
|
||||
if view is not None
|
||||
]
|
||||
pages = [view for page_item in (raw_pages if isinstance(raw_pages, list) else []) if isinstance(page_item, dict) for view in [_wiki_page_view(page_item)] if view is not None]
|
||||
return {**result, "pages": pages}
|
||||
|
||||
|
||||
@@ -2650,11 +2879,7 @@ def _public_wiki_index_item(value: Any) -> dict[str, Any] | None:
|
||||
"slug": slug,
|
||||
"title": str(value.get("title") or slug),
|
||||
"summary": str(value.get("summary") or ""),
|
||||
"category_path": (
|
||||
[str(item) for item in value.get("category_path", []) if str(item).strip()]
|
||||
if isinstance(value.get("category_path"), list)
|
||||
else []
|
||||
),
|
||||
"category_path": ([str(item) for item in value.get("category_path", []) if str(item).strip()] if isinstance(value.get("category_path"), list) else []),
|
||||
"wiki_path": str(value.get("wiki_path") or ""),
|
||||
"depth": int(value.get("depth") or 0),
|
||||
}
|
||||
@@ -2680,20 +2905,12 @@ async def _all_public_wiki_index(client: WeKnoraClient, remote_id: str) -> dict[
|
||||
version = max(version, int(payload.get("version") or 0))
|
||||
raw_groups = payload.get("groups") if isinstance(payload.get("groups"), list) else []
|
||||
group = next(
|
||||
(
|
||||
value
|
||||
for value in raw_groups
|
||||
if isinstance(value, dict) and value.get("type") == page_type
|
||||
),
|
||||
(value for value in raw_groups if isinstance(value, dict) and value.get("type") == page_type),
|
||||
{},
|
||||
)
|
||||
total = max(total, int(group.get("total") or 0))
|
||||
raw_items = group.get("items") if isinstance(group.get("items"), list) else []
|
||||
items.extend(
|
||||
item
|
||||
for value in raw_items
|
||||
if (item := _public_wiki_index_item(value)) is not None
|
||||
)
|
||||
items.extend(item for value in raw_items if (item := _public_wiki_index_item(value)) is not None)
|
||||
next_cursor = str(group.get("next_cursor") or "")
|
||||
if not next_cursor or next_cursor in seen_cursors:
|
||||
break
|
||||
@@ -3201,12 +3418,7 @@ async def search_llmwiki(request: Request, body: SearchRequest) -> dict[str, Any
|
||||
except WikiIndexError as exc:
|
||||
raise HTTPException(status_code=exc.status_code, detail={"code": exc.code, "message": str(exc)}) from None
|
||||
mapping_by_id = {str(row["id"]): row for row in rows}
|
||||
result["results"] = [
|
||||
_sanitize_conversation_deposit_value(item)
|
||||
if _is_conversation_deposit_mapping(mapping_by_id.get(str(item.get("knowledge_base_id") or ""), {}))
|
||||
else item
|
||||
for item in result.get("results") or []
|
||||
]
|
||||
result["results"] = [_sanitize_conversation_deposit_value(item) if _is_conversation_deposit_mapping(mapping_by_id.get(str(item.get("knowledge_base_id") or ""), {})) else item for item in result.get("results") or []]
|
||||
return result
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from app.gateway import deps
|
||||
from app.gateway.routers import artifacts
|
||||
|
||||
|
||||
class _FakeThreadStore:
|
||||
def __init__(self, rows: list[dict]) -> None:
|
||||
self.rows = rows
|
||||
self.calls: list[dict] = []
|
||||
|
||||
async def search(self, **kwargs) -> list[dict]:
|
||||
self.calls.append(kwargs)
|
||||
offset = int(kwargs.get("offset", 0))
|
||||
limit = int(kwargs.get("limit", 100))
|
||||
return self.rows[offset : offset + limit]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_artifact_library_aggregates_generated_non_media_files(tmp_path: Path, monkeypatch):
|
||||
thread_one = tmp_path / "thread-one" / "outputs"
|
||||
thread_two = tmp_path / "thread-two" / "outputs"
|
||||
thread_one.mkdir(parents=True)
|
||||
thread_two.mkdir(parents=True)
|
||||
|
||||
report = thread_one / "研判报告.docx"
|
||||
table = thread_one / "数据汇总.xlsx"
|
||||
image = thread_one / "配图.png"
|
||||
audio = thread_two / "录音.mp3"
|
||||
notes = thread_two / "结论.md"
|
||||
archive = thread_two / "证据.zip"
|
||||
for path, content in (
|
||||
(report, b"doc"),
|
||||
(table, b"sheet"),
|
||||
(image, b"image"),
|
||||
(audio, b"audio"),
|
||||
(notes, b"# notes"),
|
||||
(archive, b"zip"),
|
||||
):
|
||||
path.write_bytes(content)
|
||||
|
||||
os.utime(report, (100, 100))
|
||||
os.utime(table, (200, 200))
|
||||
os.utime(notes, (300, 300))
|
||||
os.utime(archive, (400, 400))
|
||||
|
||||
rows = [
|
||||
{
|
||||
"thread_id": "thread-one",
|
||||
"display_name": "战场环境综合研判",
|
||||
"metadata": {"agent_id": "analysis-agent"},
|
||||
},
|
||||
{
|
||||
"thread_id": "thread-two",
|
||||
"display_name": "生成发展报告",
|
||||
"metadata": {},
|
||||
},
|
||||
]
|
||||
store = _FakeThreadStore(rows)
|
||||
|
||||
class _Paths:
|
||||
def sandbox_outputs_dir(self, thread_id: str, *, user_id: str) -> Path:
|
||||
assert user_id == "owner"
|
||||
return tmp_path / thread_id / "outputs"
|
||||
|
||||
async def _resolve_owner(_thread_id: str) -> str:
|
||||
return "owner"
|
||||
|
||||
monkeypatch.setattr(artifacts, "get_paths", lambda: _Paths())
|
||||
monkeypatch.setattr(artifacts, "aresolve_path_user_id", _resolve_owner)
|
||||
|
||||
items = await artifacts._scan_artifact_library(store)
|
||||
|
||||
assert [item["name"] for item in items] == ["证据.zip", "结论.md", "数据汇总.xlsx", "研判报告.docx"]
|
||||
assert [item["file_kind"] for item in items] == ["archive", "text_data", "spreadsheet", "document"]
|
||||
assert items[-1]["thread_title"] == "战场环境综合研判"
|
||||
assert items[-1]["agent_id"] == "analysis-agent"
|
||||
assert all(item["name"] not in {"配图.png", "录音.mp3"} for item in items)
|
||||
assert store.calls[0]["exclude_system"] is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_artifact_library_endpoint_filters_searches_and_pages(tmp_path: Path, monkeypatch):
|
||||
output_dir = tmp_path / "thread-one" / "outputs"
|
||||
output_dir.mkdir(parents=True)
|
||||
(output_dir / "任务报告.md").write_text("report", encoding="utf-8")
|
||||
(output_dir / "任务数据.csv").write_text("a,b", encoding="utf-8")
|
||||
|
||||
store = _FakeThreadStore(
|
||||
[
|
||||
{
|
||||
"thread_id": "thread-one",
|
||||
"display_name": "任务研判对话",
|
||||
"metadata": {},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
class _Paths:
|
||||
def sandbox_outputs_dir(self, thread_id: str, *, user_id: str) -> Path:
|
||||
return tmp_path / thread_id / "outputs"
|
||||
|
||||
async def _resolve_owner(_thread_id: str) -> str:
|
||||
return "owner"
|
||||
|
||||
monkeypatch.setattr(artifacts, "get_paths", lambda: _Paths())
|
||||
monkeypatch.setattr(artifacts, "aresolve_path_user_id", _resolve_owner)
|
||||
monkeypatch.setattr(artifacts, "get_effective_user_id", lambda: "owner")
|
||||
monkeypatch.setattr(deps, "get_thread_store", lambda _request: store)
|
||||
artifacts._artifact_library_cache.clear()
|
||||
|
||||
raw_endpoint = artifacts.list_artifact_library.__wrapped__
|
||||
response = await raw_endpoint(
|
||||
request=SimpleNamespace(),
|
||||
query="研判对话",
|
||||
file_kind="text_data",
|
||||
offset=0,
|
||||
limit=1,
|
||||
refresh=True,
|
||||
)
|
||||
|
||||
assert response.total == 1
|
||||
assert response.offset == 0
|
||||
assert response.limit == 1
|
||||
assert [item.name for item in response.items] == ["任务报告.md"]
|
||||
|
||||
spreadsheet_response = await raw_endpoint(
|
||||
request=SimpleNamespace(),
|
||||
query="任务数据",
|
||||
file_kind="spreadsheet",
|
||||
offset=0,
|
||||
limit=50,
|
||||
refresh=False,
|
||||
)
|
||||
assert spreadsheet_response.total == 1
|
||||
assert spreadsheet_response.items[0].name == "任务数据.csv"
|
||||
@@ -68,7 +68,7 @@ def _published_row() -> dict[str, object]:
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_public_knowledge_base_list_exposes_only_published_wiki_identity() -> None:
|
||||
async def test_public_knowledge_base_list_exposes_live_safe_metadata_and_generated_index(monkeypatch) -> None:
|
||||
store = PublicListStore(
|
||||
[
|
||||
{
|
||||
@@ -83,21 +83,116 @@ async def test_public_knowledge_base_list_exposes_only_published_wiki_identity()
|
||||
"owner_user_id": "system",
|
||||
"name": "对话沉淀",
|
||||
},
|
||||
{
|
||||
**_published_row(),
|
||||
"id": "stale-kb",
|
||||
"weknora_id": "remote-missing",
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
class Client:
|
||||
async def list_knowledge_bases(self):
|
||||
return [
|
||||
{
|
||||
"id": "remote-one",
|
||||
"name": "远端公开知识库",
|
||||
"description": "远端公开说明",
|
||||
"type": "wiki",
|
||||
"knowledge_count": 12,
|
||||
"chunk_count": 34,
|
||||
"processing_count": 1,
|
||||
"share_count": 3,
|
||||
"created_at": "2026-07-01T08:00:00+08:00",
|
||||
"updated_at": "2026-08-01T08:00:00+08:00",
|
||||
"tenant_id": 10001,
|
||||
"creator_id": "secret-creator",
|
||||
"storage_config": {"secret_key": "must-not-leak"},
|
||||
}
|
||||
]
|
||||
|
||||
async def get_knowledge_base(self, knowledge_base_id: str):
|
||||
assert knowledge_base_id == "remote-one"
|
||||
return {
|
||||
"id": knowledge_base_id,
|
||||
"vector_store_name": "System default",
|
||||
"vector_store_source": "env",
|
||||
"vector_store_engine_type": "elasticsearch",
|
||||
"vector_store_status": "available",
|
||||
"wiki_config": {"enabled": True},
|
||||
}
|
||||
|
||||
async def list_wiki_pages(self, knowledge_base_id: str, **kwargs):
|
||||
assert knowledge_base_id == "remote-one"
|
||||
assert kwargs == {"page": 1, "page_size": 500, "query": ""}
|
||||
return {
|
||||
"pages": [
|
||||
{
|
||||
"id": "page-one",
|
||||
"slug": "concept/start",
|
||||
"title": "开始使用",
|
||||
"page_type": "concept",
|
||||
"status": "completed",
|
||||
"created_at": "2026-07-02T08:00:00+08:00",
|
||||
"updated_at": "2026-08-02T08:00:00+08:00",
|
||||
},
|
||||
{
|
||||
"id": "page-two",
|
||||
"slug": "summary/overview",
|
||||
"title": "知识概览",
|
||||
"page_type": "summary",
|
||||
"status": "completed",
|
||||
"created_at": "2026-07-03T08:00:00+08:00",
|
||||
"updated_at": "2026-08-03T08:00:00+08:00",
|
||||
},
|
||||
],
|
||||
"total": 2,
|
||||
"page": 1,
|
||||
"page_size": 500,
|
||||
"total_pages": 1,
|
||||
}
|
||||
|
||||
monkeypatch.setattr(llmwiki, "_client_or_503", lambda: Client())
|
||||
|
||||
payload = await llmwiki.list_public_knowledge_bases(_request(store))
|
||||
|
||||
assert store.calls == [("__public_wiki__", "public", False)]
|
||||
assert payload == {
|
||||
"knowledge_bases": [
|
||||
{
|
||||
"id": "1063edee-332a-43b4-bbd9-73bf0a708b94",
|
||||
"name": "公开知识库",
|
||||
}
|
||||
],
|
||||
"total": 1,
|
||||
assert payload["total"] == 1
|
||||
item = payload["knowledge_bases"][0]
|
||||
assert item["id"] == "1063edee-332a-43b4-bbd9-73bf0a708b94"
|
||||
assert item["name"] == "远端公开知识库"
|
||||
assert item["description"] == "远端公开说明"
|
||||
assert item["type"] == "wiki"
|
||||
assert item["knowledge_count"] == 12
|
||||
assert item["chunk_count"] == 34
|
||||
assert item["processing_count"] == 1
|
||||
assert item["share_count"] == 3
|
||||
assert item["is_processing"] is True
|
||||
assert item["remote_created_at"] == "2026-07-01T08:00:00+08:00"
|
||||
assert item["remote_updated_at"] == "2026-08-01T08:00:00+08:00"
|
||||
assert item["vector_store_engine_type"] == "elasticsearch"
|
||||
assert item["wiki"] == {
|
||||
"status": "ready",
|
||||
"page_count": 2,
|
||||
"loaded_page_count": 2,
|
||||
"stats_complete": True,
|
||||
"page_type_counts": {"concept": 1, "summary": 1},
|
||||
"page_status_counts": {"completed": 2},
|
||||
"index_mode": "generated",
|
||||
"has_explicit_index": False,
|
||||
"index_slug": "__index__",
|
||||
"default_slug": "__index__",
|
||||
"created_at": "2026-07-02T08:00:00+08:00",
|
||||
"updated_at": "2026-08-03T08:00:00+08:00",
|
||||
"embed_route": "/embed/knowledge/1063edee-332a-43b4-bbd9-73bf0a708b94?wikiSlug=__index__",
|
||||
}
|
||||
assert item["local_index"]["state"] == "not_synced"
|
||||
assert item["local_index"]["fully_vectorized"] is False
|
||||
assert item["owner_user_id"] is None
|
||||
assert "weknora_id" not in item
|
||||
assert "tenant_id" not in item
|
||||
assert "creator_id" not in item
|
||||
assert "storage_config" not in item
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -178,6 +273,35 @@ async def test_public_wiki_page_list_is_normalized(monkeypatch) -> None:
|
||||
assert payload["pages"][0]["aliases"] == []
|
||||
|
||||
|
||||
def test_public_wiki_summary_prefers_an_explicit_index_page() -> None:
|
||||
payload = llmwiki._public_wiki_summary(
|
||||
"mapping-one",
|
||||
[
|
||||
{
|
||||
"slug": "guide/start",
|
||||
"title": "开始使用",
|
||||
"page_type": "article",
|
||||
"status": "completed",
|
||||
},
|
||||
{
|
||||
"slug": "home",
|
||||
"title": "知识首页",
|
||||
"page_type": "index",
|
||||
"status": "completed",
|
||||
},
|
||||
],
|
||||
total=2,
|
||||
complete=True,
|
||||
)
|
||||
|
||||
assert payload["status"] == "ready"
|
||||
assert payload["index_mode"] == "explicit"
|
||||
assert payload["has_explicit_index"] is True
|
||||
assert payload["index_slug"] == "home"
|
||||
assert payload["default_slug"] == "home"
|
||||
assert payload["embed_route"] == "/embed/knowledge/mapping-one?wikiSlug=home"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_public_wiki_file_is_authorized_and_streamed(monkeypatch) -> None:
|
||||
store = PublicStore(_published_row())
|
||||
|
||||
Reference in new issue
Block a user