feat: add artifact library and improve public wiki
This commit is contained in:
1 parent
1b43205c1b
commit
dd0ee282df
16 files changed
+4081
-98
No files matched your search
@@ -1,11 +1,15 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import mimetypes
|
||||
import time
|
||||
import zipfile
|
||||
from datetime import datetime, timezone
|
||||
from collections import OrderedDict
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
from urllib.parse import quote
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from fastapi import APIRouter, HTTPException, Query, Request
|
||||
from fastapi.responses import FileResponse, PlainTextResponse, Response
|
||||
from pydantic import BaseModel
|
||||
|
||||
@@ -13,6 +17,7 @@ from app.gateway.authz import require_permission
|
||||
from app.gateway.path_utils import aresolve_thread_virtual_path
|
||||
from deerflow.config.paths import get_paths
|
||||
from deerflow.runtime.thread_paths import aresolve_path_user_id
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -38,6 +43,84 @@ EDITABLE_TEXT_SUFFIXES = {
|
||||
|
||||
_VIRTUAL_USER_DATA_PREFIX = "mnt/user-data"
|
||||
|
||||
ArtifactLibraryKind = Literal["all", "document", "spreadsheet", "text_data", "archive", "other"]
|
||||
|
||||
_MEDIA_MIME_PREFIXES = ("image/", "audio/", "video/")
|
||||
_MEDIA_SUFFIXES = {
|
||||
".aac",
|
||||
".aiff",
|
||||
".ape",
|
||||
".avi",
|
||||
".bmp",
|
||||
".flac",
|
||||
".gif",
|
||||
".heic",
|
||||
".ico",
|
||||
".jpeg",
|
||||
".jpg",
|
||||
".m4a",
|
||||
".m4v",
|
||||
".mkv",
|
||||
".mov",
|
||||
".mp3",
|
||||
".mp4",
|
||||
".ogg",
|
||||
".png",
|
||||
".svg",
|
||||
".tiff",
|
||||
".wav",
|
||||
".webm",
|
||||
".webp",
|
||||
".wma",
|
||||
}
|
||||
_DOCUMENT_SUFFIXES = {
|
||||
".doc",
|
||||
".docx",
|
||||
".epub",
|
||||
".odt",
|
||||
".pdf",
|
||||
".ppt",
|
||||
".pptx",
|
||||
".rtf",
|
||||
}
|
||||
_SPREADSHEET_SUFFIXES = {".csv", ".ods", ".tsv", ".xls", ".xlsx"}
|
||||
_TEXT_DATA_SUFFIXES = {
|
||||
".bash",
|
||||
".c",
|
||||
".conf",
|
||||
".cpp",
|
||||
".css",
|
||||
".go",
|
||||
".h",
|
||||
".html",
|
||||
".ini",
|
||||
".java",
|
||||
".js",
|
||||
".json",
|
||||
".jsonl",
|
||||
".log",
|
||||
".markdown",
|
||||
".md",
|
||||
".mmd",
|
||||
".py",
|
||||
".rs",
|
||||
".sh",
|
||||
".sql",
|
||||
".toml",
|
||||
".ts",
|
||||
".tsx",
|
||||
".txt",
|
||||
".xml",
|
||||
".yaml",
|
||||
".yml",
|
||||
}
|
||||
_ARCHIVE_SUFFIXES = {".7z", ".gz", ".rar", ".skill", ".tar", ".tgz", ".zip"}
|
||||
_ARTIFACT_LIBRARY_CACHE_TTL_SECONDS = 15.0
|
||||
_ARTIFACT_LIBRARY_CACHE_MAX_USERS = 128
|
||||
_ARTIFACT_LIBRARY_SCAN_CONCURRENCY = 8
|
||||
_ARTIFACT_LIBRARY_THREAD_BATCH_SIZE = 500
|
||||
_artifact_library_cache: OrderedDict[str, tuple[float, list[dict]]] = OrderedDict()
|
||||
|
||||
|
||||
def coerce_artifact_virtual_path(path: str) -> str:
|
||||
"""Normalize an artifacts URL path to a sandbox virtual path.
|
||||
@@ -60,6 +143,26 @@ def coerce_artifact_virtual_path(path: str) -> str:
|
||||
|
||||
class ArtifactUpdateRequest(BaseModel):
|
||||
content: str
|
||||
|
||||
|
||||
class ArtifactLibraryItem(BaseModel):
|
||||
thread_id: str
|
||||
thread_title: str
|
||||
agent_id: str | None = None
|
||||
thread_type: str | None = None
|
||||
path: str
|
||||
name: str
|
||||
size_bytes: int
|
||||
mime_type: str | None = None
|
||||
modified_at: str
|
||||
file_kind: str
|
||||
|
||||
|
||||
class ArtifactLibraryResponse(BaseModel):
|
||||
total: int
|
||||
offset: int
|
||||
limit: int
|
||||
items: list[ArtifactLibraryItem]
|
||||
|
||||
|
||||
def _build_content_disposition(disposition_type: str, filename: str) -> str:
|
||||
@@ -139,21 +242,184 @@ def collect_outputs_listing(outputs_dir: Path) -> list[dict]:
|
||||
resolved.relative_to(outputs_root)
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
relative = resolved.relative_to(outputs_root).as_posix()
|
||||
mime_type, _ = mimetypes.guess_type(resolved.name)
|
||||
stat = resolved.stat()
|
||||
try:
|
||||
relative = resolved.relative_to(outputs_root).as_posix()
|
||||
mime_type, _ = mimetypes.guess_type(resolved.name)
|
||||
stat = resolved.stat()
|
||||
except OSError:
|
||||
# A run may atomically replace or remove a file while this directory
|
||||
# is being scanned. Treat that transient entry as absent instead of
|
||||
# failing the complete thread/library listing.
|
||||
continue
|
||||
files.append(
|
||||
{
|
||||
"path": f"mnt/user-data/outputs/{relative}",
|
||||
"name": resolved.name,
|
||||
"size_bytes": stat.st_size,
|
||||
"mime_type": mime_type,
|
||||
"modified_at": datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc).isoformat(),
|
||||
"modified_at": datetime.fromtimestamp(stat.st_mtime, tz=UTC).isoformat(),
|
||||
}
|
||||
)
|
||||
|
||||
files.sort(key=lambda f: f["modified_at"], reverse=True)
|
||||
return files
|
||||
|
||||
|
||||
def artifact_library_kind(name: str) -> str:
|
||||
"""Classify one non-media artifact for the library filters."""
|
||||
suffix = Path(name).suffix.lower()
|
||||
if suffix in _DOCUMENT_SUFFIXES:
|
||||
return "document"
|
||||
if suffix in _SPREADSHEET_SUFFIXES:
|
||||
return "spreadsheet"
|
||||
if suffix in _TEXT_DATA_SUFFIXES:
|
||||
return "text_data"
|
||||
if suffix in _ARCHIVE_SUFFIXES:
|
||||
return "archive"
|
||||
return "other"
|
||||
|
||||
|
||||
def is_artifact_library_file(file: dict) -> bool:
|
||||
"""Return whether a generated file belongs in the document-only library."""
|
||||
mime_type = str(file.get("mime_type") or "").lower()
|
||||
if mime_type.startswith(_MEDIA_MIME_PREFIXES):
|
||||
return False
|
||||
return Path(str(file.get("name") or file.get("path") or "")).suffix.lower() not in _MEDIA_SUFFIXES
|
||||
|
||||
|
||||
async def _search_artifact_library_threads(thread_store) -> list[dict]:
|
||||
"""Load every normal conversation owned by the current request user."""
|
||||
rows: list[dict] = []
|
||||
seen: set[str] = set()
|
||||
offset = 0
|
||||
while True:
|
||||
batch = await thread_store.search(
|
||||
limit=_ARTIFACT_LIBRARY_THREAD_BATCH_SIZE,
|
||||
offset=offset,
|
||||
exclude_system=True,
|
||||
)
|
||||
if not batch:
|
||||
break
|
||||
added = 0
|
||||
for row in batch:
|
||||
thread_id = str(row.get("thread_id") or "")
|
||||
if not thread_id or thread_id in seen:
|
||||
continue
|
||||
seen.add(thread_id)
|
||||
rows.append(row)
|
||||
added += 1
|
||||
if len(batch) < _ARTIFACT_LIBRARY_THREAD_BATCH_SIZE or added == 0:
|
||||
break
|
||||
offset += len(batch)
|
||||
return rows
|
||||
|
||||
|
||||
async def _scan_artifact_library(thread_store) -> list[dict]:
|
||||
"""Scan generated outputs for all current-user conversation threads."""
|
||||
rows = await _search_artifact_library_threads(thread_store)
|
||||
paths = get_paths()
|
||||
semaphore = asyncio.Semaphore(_ARTIFACT_LIBRARY_SCAN_CONCURRENCY)
|
||||
|
||||
async def scan_thread(row: dict) -> list[dict]:
|
||||
thread_id = str(row.get("thread_id") or "")
|
||||
if not thread_id:
|
||||
return []
|
||||
async with semaphore:
|
||||
try:
|
||||
path_user_id = await aresolve_path_user_id(thread_id)
|
||||
outputs_dir = paths.sandbox_outputs_dir(thread_id, user_id=path_user_id)
|
||||
files = await asyncio.to_thread(collect_outputs_listing, outputs_dir)
|
||||
except Exception:
|
||||
logger.warning("Failed to scan artifacts for library thread %s", thread_id, exc_info=True)
|
||||
return []
|
||||
|
||||
metadata = row.get("metadata") if isinstance(row.get("metadata"), dict) else {}
|
||||
agent_id = metadata.get("agent_id")
|
||||
thread_type = metadata.get("thread_type")
|
||||
title = str(row.get("display_name") or "新对话")
|
||||
items: list[dict] = []
|
||||
for file in files:
|
||||
if not is_artifact_library_file(file):
|
||||
continue
|
||||
items.append(
|
||||
{
|
||||
"thread_id": thread_id,
|
||||
"thread_title": title,
|
||||
"agent_id": agent_id if isinstance(agent_id, str) and agent_id else None,
|
||||
"thread_type": thread_type if isinstance(thread_type, str) and thread_type else None,
|
||||
**file,
|
||||
"file_kind": artifact_library_kind(str(file.get("name") or "")),
|
||||
}
|
||||
)
|
||||
return items
|
||||
|
||||
nested = await asyncio.gather(*(scan_thread(row) for row in rows))
|
||||
items = [item for group in nested for item in group]
|
||||
items.sort(
|
||||
key=lambda item: (
|
||||
str(item.get("modified_at") or ""),
|
||||
str(item.get("thread_id") or ""),
|
||||
str(item.get("path") or ""),
|
||||
),
|
||||
reverse=True,
|
||||
)
|
||||
return items
|
||||
|
||||
|
||||
async def _cached_artifact_library(thread_store, *, refresh: bool) -> list[dict]:
|
||||
user_id = str(get_effective_user_id() or "default")
|
||||
now = time.monotonic()
|
||||
cached = _artifact_library_cache.get(user_id)
|
||||
if not refresh and cached and cached[0] > now:
|
||||
_artifact_library_cache.move_to_end(user_id)
|
||||
return cached[1]
|
||||
|
||||
items = await _scan_artifact_library(thread_store)
|
||||
_artifact_library_cache[user_id] = (now + _ARTIFACT_LIBRARY_CACHE_TTL_SECONDS, items)
|
||||
_artifact_library_cache.move_to_end(user_id)
|
||||
while len(_artifact_library_cache) > _ARTIFACT_LIBRARY_CACHE_MAX_USERS:
|
||||
_artifact_library_cache.popitem(last=False)
|
||||
return items
|
||||
|
||||
|
||||
@router.get(
|
||||
"/artifact-library",
|
||||
response_model=ArtifactLibraryResponse,
|
||||
summary="List Generated Files Across Conversations",
|
||||
description="Aggregate non-media files generated in the current user's normal conversations.",
|
||||
)
|
||||
@require_permission("threads", "read")
|
||||
async def list_artifact_library(
|
||||
request: Request,
|
||||
query: str | None = Query(default=None, max_length=200),
|
||||
file_kind: ArtifactLibraryKind = Query(default="all"),
|
||||
offset: int = Query(default=0, ge=0),
|
||||
limit: int = Query(default=50, ge=1, le=200),
|
||||
refresh: bool = Query(default=False),
|
||||
) -> ArtifactLibraryResponse:
|
||||
"""Return a flat, newest-first library of generated conversation files.
|
||||
|
||||
The filesystem remains the source of truth: every owned, non-system thread's
|
||||
``outputs`` directory is scanned, including files written without an
|
||||
explicit ``present_files`` call. Uploads and image/audio/video outputs are
|
||||
intentionally outside this collection.
|
||||
"""
|
||||
from app.gateway.deps import get_thread_store
|
||||
|
||||
items = await _cached_artifact_library(get_thread_store(request), refresh=refresh)
|
||||
normalized_query = (query or "").strip().casefold()
|
||||
filtered = [
|
||||
item
|
||||
for item in items
|
||||
if (file_kind == "all" or item["file_kind"] == file_kind) and (not normalized_query or normalized_query in str(item.get("name") or "").casefold() or normalized_query in str(item.get("thread_title") or "").casefold())
|
||||
]
|
||||
page = filtered[offset : offset + limit]
|
||||
return ArtifactLibraryResponse(
|
||||
total=len(filtered),
|
||||
offset=offset,
|
||||
limit=limit,
|
||||
items=[ArtifactLibraryItem.model_validate(item) for item in page],
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
|
||||
Reference in new issue
Block a user