sync: 鍚屾灞€鍩熺綉鏈€鏂颁唬鐮?(source master @ b69568e4)

This commit is contained in:
gozeros committed 2026-10-21 14:51:31 +08:00
1 parent 4f7f07504a
commit 8bc8b0e082
1651 files changed
+278599 -11416

No files matched your search

@@ -1,9 +1,8 @@
import asyncio
import logging
import mimetypes
import time
import re
import zipfile
from collections import OrderedDict
from datetime import UTC, datetime
from pathlib import Path
from typing import Literal
@@ -11,7 +10,7 @@ from urllib.parse import quote
from fastapi import APIRouter, HTTPException, Query, Request
from fastapi.responses import FileResponse, PlainTextResponse, Response
from pydantic import BaseModel
from pydantic import BaseModel, Field
from app.gateway.authz import require_permission
from app.gateway.path_utils import aresolve_thread_virtual_path
@@ -44,7 +43,7 @@ EDITABLE_TEXT_SUFFIXES = {
_VIRTUAL_USER_DATA_PREFIX = "mnt/user-data"
ArtifactLibraryKind = Literal["all", "document", "spreadsheet", "text_data", "archive", "other"]
ArtifactLibrarySource = Literal["all", "general_qa", "agent"]
ArtifactLibrarySource = Literal["all", "general_qa", "agent", "strategy"]
_MEDIA_MIME_PREFIXES = ("image/", "audio/", "video/")
_MEDIA_SUFFIXES = {
@@ -116,11 +115,20 @@ _TEXT_DATA_SUFFIXES = {
".yml",
}
_ARCHIVE_SUFFIXES = {".7z", ".gz", ".rar", ".skill", ".tar", ".tgz", ".zip"}
_ARTIFACT_LIBRARY_CACHE_TTL_SECONDS = 15.0
_ARTIFACT_LIBRARY_CACHE_MAX_USERS = 128
_ARTIFACT_LIBRARY_SCAN_CONCURRENCY = 8
_ARTIFACT_LIBRARY_INDEX_TTL_SECONDS = 300.0
# Filesystem reconciliation is deliberately low-concurrency: it is a background
# correctness sweep, not work that should compete with active model runs.
_ARTIFACT_LIBRARY_SCAN_CONCURRENCY = 2
_ARTIFACT_LIBRARY_THREAD_BATCH_SIZE = 500
_artifact_library_cache: OrderedDict[str, tuple[float, list[dict]]] = OrderedDict()
_ARTIFACT_LIBRARY_SCAN_BATCH_SIZE = 100
_STRATEGY_PROJECT_BATCH_SIZE = 200
_STRATEGY_PROJECT_ID_PREFIX = "strategy-selection-"
_STRATEGY_PROJECT_SNAPSHOT_KIND = "strategy-selection-project"
_STRATEGY_PROJECT_THREAD_TYPE = "strategy_selection"
class ArtifactLibraryScanIncomplete(RuntimeError):
"""Raised when a reconciliation cannot safely publish a complete generation."""
def coerce_artifact_virtual_path(path: str) -> str:
@@ -152,7 +160,12 @@ class ArtifactLibraryItem(BaseModel):
agent_id: str | None = None
agent_name: str | None = None
thread_type: str | None = None
source_type: Literal["general_qa", "agent"]
source_type: Literal["general_qa", "agent", "strategy"]
strategy_project_id: str | None = None
strategy_name: str | None = None
strategy_stale: bool = False
strategy_source_kind: Literal["strategy_selection", "analysis_project"] | None = None
download_url: str | None = None
path: str
name: str
size_bytes: int
@@ -166,6 +179,23 @@ class ArtifactLibraryResponse(BaseModel):
offset: int
limit: int
items: list[ArtifactLibraryItem]
sync_status: Literal["idle", "syncing", "ready", "error"] = "idle"
syncing: bool = False
indexed_at: str | None = None
sync_error: str | None = None
class ArtifactPreviewResponse(BaseModel):
"""A bounded text preview that never requires loading the complete file."""
path: str
name: str
total_bytes: int
content: str
truncated: bool
strategy: str
stats: dict | None = None
warnings: list[str] = Field(default_factory=list)
def _build_content_disposition(disposition_type: str, filename: str) -> str:
@@ -224,7 +254,11 @@ def _extract_file_from_skill_archive(zip_path: Path, internal_path: str) -> byte
return None
def collect_outputs_listing(outputs_dir: Path) -> list[dict]:
def collect_outputs_listing(
outputs_dir: Path,
*,
errors: list[str] | None = None,
) -> list[dict]:
"""Walk an outputs directory and return artifact listing entries.
Each entry has ``path`` (virtual ``mnt/user-data/outputs/...``), ``name``,
@@ -251,8 +285,11 @@ def collect_outputs_listing(outputs_dir: Path) -> list[dict]:
stat = resolved.stat()
except OSError:
# A run may atomically replace or remove a file while this directory
# is being scanned. Treat that transient entry as absent instead of
# failing the complete thread/library listing.
# is being scanned. Direct per-thread listings skip that transient
# entry; full-library reconciliation also records it and refuses to
# replace the last complete database generation.
if errors is not None:
errors.append(entry.name)
continue
files.append(
{
@@ -290,8 +327,154 @@ def is_artifact_library_file(file: dict) -> bool:
return Path(str(file.get("name") or file.get("path") or "")).suffix.lower() not in _MEDIA_SUFFIXES
async def _search_artifact_library_threads(thread_store) -> list[dict]:
def _request_user_id(request: Request) -> str:
"""Match the owner resolution used by the parallel-task router."""
user = getattr(getattr(request, "state", None), "user", None)
if user is not None and getattr(user, "id", None) is not None:
return str(user.id)
return str(get_effective_user_id() or "default")
def _parallel_agent_task_store(request: Request, *, required: bool = False):
state = getattr(getattr(request, "app", None), "state", None)
store = getattr(state, "parallel_agent_task_store", None)
if store is None and required:
raise HTTPException(status_code=503, detail="Strategy project store not available")
return store
def _artifact_library_store(request: Request):
state = getattr(getattr(request, "app", None), "state", None)
store = getattr(state, "artifact_library_store", None)
if store is None:
raise HTTPException(status_code=503, detail="Artifact library index not available")
return store
def _strategy_project_document(task: dict) -> dict | None:
"""Return the completed strategy document saved inside one task snapshot.
Strategy-selection projects predate a dedicated artifact table. Their
server record is a ``parallel_agent_tasks`` row whose opaque ``messages``
list contains one versioned snapshot envelope. Keep parsing strict so an
unrelated parallel task cannot appear in the library merely by carrying a
``synthesis``-looking field.
"""
project_id = str(task.get("id") or "").strip()
if not project_id.startswith(_STRATEGY_PROJECT_ID_PREFIX):
return None
messages = task.get("messages")
if not isinstance(messages, list):
return None
envelope = next(
(
item
for item in messages
if isinstance(item, dict)
and item.get("kind") == _STRATEGY_PROJECT_SNAPSHOT_KIND
and item.get("version") == 1
and isinstance(item.get("snapshot"), dict)
),
None,
)
if envelope is None:
return None
snapshot = envelope["snapshot"]
if snapshot.get("version") != 1:
return None
content = snapshot.get("synthesis")
if snapshot.get("synthesisStatus") != "done" or not isinstance(content, str) or not content.strip():
return None
task_draft = snapshot.get("taskDraft") if isinstance(snapshot.get("taskDraft"), dict) else {}
strategy_name = str(task_draft.get("name") or task.get("title") or "").strip() or "未命名策略项目"
return {
"project_id": project_id,
"strategy_name": strategy_name,
"content": content,
# A changed task deliberately retains its last completed synthesis for
# historical viewing/download until the replacement run succeeds.
"strategy_stale": snapshot.get("workflowStale") is True,
"modified_at": str(task.get("updated_at") or task.get("created_at") or ""),
}
def _strategy_artifact_filename(strategy_name: str) -> str:
clean = re.sub(r'[<>:"/\\|?*\x00-\x1f]+', "_", strategy_name).strip(" .")[:120]
return f"{clean or '策略优选方案'}.md"
def _strategy_project_artifact_item(task: dict) -> dict | None:
document = _strategy_project_document(task)
if document is None:
return None
project_id = document["project_id"]
encoded_project_id = quote(project_id, safe="")
name = _strategy_artifact_filename(document["strategy_name"])
return {
# Keep the legacy identity fields populated for stable table keys while
# the explicit strategy fields identify the non-thread source.
"thread_id": project_id,
"thread_title": document["strategy_name"],
"agent_id": None,
"agent_name": None,
"thread_type": _STRATEGY_PROJECT_THREAD_TYPE,
"source_type": "strategy",
"strategy_project_id": project_id,
"strategy_name": document["strategy_name"],
"strategy_stale": document["strategy_stale"],
"strategy_source_kind": "strategy_selection",
"download_url": f"/api/strategy-project-artifacts/{encoded_project_id}/download",
"path": f"strategy-projects/{encoded_project_id}/solution.md",
"name": name,
"size_bytes": len(document["content"].encode("utf-8")),
"mime_type": "text/markdown",
"modified_at": document["modified_at"],
"file_kind": "text_data",
}
async def _scan_strategy_project_artifacts(
strategy_task_store,
*,
user_id: str | None = None,
strict: bool = False,
) -> list[dict]:
"""Project completed syntheses into the otherwise file-backed library."""
if strategy_task_store is None:
return []
resolved_user_id = user_id or str(get_effective_user_id() or "default")
rows: list[dict] = []
offset = 0
try:
while True:
batch = await strategy_task_store.list_tasks(
resolved_user_id,
limit=_STRATEGY_PROJECT_BATCH_SIZE,
offset=offset,
id_prefix=_STRATEGY_PROJECT_ID_PREFIX,
include_messages=True,
)
rows.extend(batch)
if len(batch) < _STRATEGY_PROJECT_BATCH_SIZE:
break
offset += len(batch)
except Exception as exc:
logger.warning("Failed to load strategy project artifacts", exc_info=True)
if strict:
raise ArtifactLibraryScanIncomplete(
"Strategy artifact source could not be read completely"
) from exc
return []
return [item for row in rows if (item := _strategy_project_artifact_item(row)) is not None]
async def _search_artifact_library_threads(
thread_store,
*,
user_id: str | None = None,
) -> list[dict]:
"""Load every normal conversation owned by the current request user."""
resolved_user_id = user_id or str(get_effective_user_id() or "default")
rows: list[dict] = []
seen: set[str] = set()
offset = 0
@@ -299,6 +482,7 @@ async def _search_artifact_library_threads(thread_store) -> list[dict]:
batch = await thread_store.search(
limit=_ARTIFACT_LIBRARY_THREAD_BATCH_SIZE,
offset=offset,
user_id=resolved_user_id,
exclude_system=True,
)
if not batch:
@@ -317,9 +501,17 @@ async def _search_artifact_library_threads(thread_store) -> list[dict]:
return rows
async def _scan_artifact_library(thread_store) -> list[dict]:
"""Scan generated outputs for all current-user conversation threads."""
rows = await _search_artifact_library_threads(thread_store)
async def _scan_artifact_library(
thread_store,
strategy_task_store=None,
*,
user_id: str | None = None,
strict: bool = False,
) -> list[dict]:
"""Scan thread outputs and completed strategy-project syntheses."""
explicit_user_id = user_id
resolved_user_id = user_id or str(get_effective_user_id() or "default")
rows = await _search_artifact_library_threads(thread_store, user_id=resolved_user_id)
paths = get_paths()
semaphore = asyncio.Semaphore(_ARTIFACT_LIBRARY_SCAN_CONCURRENCY)
@@ -329,19 +521,50 @@ async def _scan_artifact_library(thread_store) -> list[dict]:
return []
async with semaphore:
try:
path_user_id = await aresolve_path_user_id(thread_id)
outputs_dir = paths.sandbox_outputs_dir(thread_id, user_id=path_user_id)
files = await asyncio.to_thread(collect_outputs_listing, outputs_dir)
except Exception:
# Background indexing passes an explicit owner from the
# authenticated request, avoiding one DB owner lookup per
# historical thread. Direct/legacy callers retain the safe
# resolver fallback.
path_user_id = (
resolved_user_id
if explicit_user_id is not None
else await aresolve_path_user_id(thread_id)
)
outputs_dir = paths.sandbox_outputs_dir(
thread_id,
user_id=path_user_id,
)
scan_errors: list[str] = []
files = await asyncio.to_thread(
collect_outputs_listing,
outputs_dir,
errors=scan_errors,
)
if strict and scan_errors:
raise ArtifactLibraryScanIncomplete(
f"Thread artifact scan observed {len(scan_errors)} transient file error(s)"
)
except Exception as exc:
logger.warning("Failed to scan artifacts for library thread %s", thread_id, exc_info=True)
if strict:
if isinstance(exc, ArtifactLibraryScanIncomplete):
raise
raise ArtifactLibraryScanIncomplete(
"A thread artifact source could not be read completely"
) from exc
return []
metadata = row.get("metadata") if isinstance(row.get("metadata"), dict) else {}
agent_id = metadata.get("agent_id")
agent_name = metadata.get("agent_name")
thread_type = metadata.get("thread_type")
analysis_project_mode = metadata.get("analysis_project_mode")
strategy_project_id = metadata.get("analysis_project_id") if analysis_project_mode == "strategy" else None
strategy_name = metadata.get("analysis_project_name") if analysis_project_mode == "strategy" else None
normalized_agent_id = agent_id.strip() if isinstance(agent_id, str) else ""
normalized_agent_name = agent_name.strip() if isinstance(agent_name, str) else ""
normalized_strategy_project_id = strategy_project_id.strip() if isinstance(strategy_project_id, str) else ""
normalized_strategy_name = strategy_name.strip() if isinstance(strategy_name, str) else ""
title = str(row.get("display_name") or "新对话")
items: list[dict] = []
for file in files:
@@ -354,15 +577,40 @@ async def _scan_artifact_library(thread_store) -> list[dict]:
"agent_id": normalized_agent_id or None,
"agent_name": normalized_agent_name or None,
"thread_type": thread_type if isinstance(thread_type, str) and thread_type else None,
"source_type": "agent" if normalized_agent_id else "general_qa",
"source_type": (
"strategy"
if analysis_project_mode == "strategy"
else ("agent" if normalized_agent_id else "general_qa")
),
"strategy_project_id": normalized_strategy_project_id or None,
"strategy_name": (
normalized_strategy_name or title
if analysis_project_mode == "strategy"
else None
),
"strategy_source_kind": "analysis_project" if analysis_project_mode == "strategy" else None,
**file,
"file_kind": artifact_library_kind(str(file.get("name") or "")),
}
)
return items
nested = await asyncio.gather(*(scan_thread(row) for row in rows))
items = [item for group in nested for item in group]
# Do not create one asyncio.Task per historical conversation. Batching keeps
# memory bounded for long-lived accounts while the semaphore limits disk IO.
items: list[dict] = []
for start in range(0, len(rows), _ARTIFACT_LIBRARY_SCAN_BATCH_SIZE):
nested = await asyncio.gather(
*(scan_thread(row) for row in rows[start : start + _ARTIFACT_LIBRARY_SCAN_BATCH_SIZE])
)
items.extend(item for group in nested for item in group)
await asyncio.sleep(0)
items.extend(
await _scan_strategy_project_artifacts(
strategy_task_store,
user_id=resolved_user_id,
strict=strict,
)
)
items.sort(
key=lambda item: (
str(item.get("modified_at") or ""),
@@ -374,27 +622,131 @@ async def _scan_artifact_library(thread_store) -> list[dict]:
return items
async def _cached_artifact_library(thread_store, *, refresh: bool) -> list[dict]:
user_id = str(get_effective_user_id() or "default")
now = time.monotonic()
cached = _artifact_library_cache.get(user_id)
if not refresh and cached and cached[0] > now:
_artifact_library_cache.move_to_end(user_id)
return cached[1]
async def _resolve_artifact_library_agent_names(
items: list[dict],
agent_store,
*,
user_id: str,
) -> list[dict]:
"""Resolve agent display names once during indexing, never on every page read."""
agent_ids = list(
dict.fromkeys(
str(item.get("agent_id") or "")
for item in items
if item.get("agent_id")
)
)
agent_names: dict[str, str] = {}
if agent_ids and agent_store is not None:
try:
agents = await agent_store.list_by_ids(agent_ids, user_id)
agent_names = {
str(agent.get("id") or ""): str(agent.get("name") or "").strip()
for agent in agents
if agent.get("id") and agent.get("name")
}
except Exception:
logger.warning("Failed to resolve artifact library agent names", exc_info=True)
return [
{
**item,
"agent_name": (
agent_names.get(str(item.get("agent_id") or ""))
or item.get("agent_name")
or item.get("agent_id")
),
}
for item in items
]
items = await _scan_artifact_library(thread_store)
_artifact_library_cache[user_id] = (now + _ARTIFACT_LIBRARY_CACHE_TTL_SECONDS, items)
_artifact_library_cache.move_to_end(user_id)
while len(_artifact_library_cache) > _ARTIFACT_LIBRARY_CACHE_MAX_USERS:
_artifact_library_cache.popitem(last=False)
return items
async def _run_artifact_library_sync(app, *, user_id: str, generation: str) -> None:
"""Reconcile filesystem/task metadata into the persisted index off-request."""
store = app.state.artifact_library_store
try:
gate = getattr(app.state, "artifact_library_sync_semaphore", None)
async def _scan_and_publish() -> bool:
items = await _scan_artifact_library(
app.state.thread_store,
getattr(app.state, "parallel_agent_task_store", None),
user_id=user_id,
strict=True,
)
enriched = await _resolve_artifact_library_agent_names(
items,
getattr(app.state, "agent_store", None),
user_id=user_id,
)
published = await store.finish_sync(user_id, generation, enriched)
if published:
logger.info(
"Artifact library index refreshed for user %s (%d items)",
user_id,
len(enriched),
)
return published
if gate is None:
published = await _scan_and_publish()
else:
async with gate:
published = await _scan_and_publish()
if published:
return
except asyncio.CancelledError:
try:
await asyncio.shield(
store.fail_sync(user_id, generation, "Artifact library refresh cancelled")
)
except Exception:
logger.debug("Failed to record cancelled artifact refresh", exc_info=True)
raise
except Exception:
logger.exception("Artifact library background refresh failed for user %s", user_id)
try:
await store.fail_sync(
user_id,
generation,
"资料索引同步失败,请稍后重试",
)
except Exception:
logger.exception("Failed to persist artifact library refresh failure")
def _schedule_artifact_library_sync(app, *, user_id: str, generation: str) -> None:
"""Start one tracked refresh task after its cross-worker DB lease is claimed."""
tasks = getattr(app.state, "artifact_library_sync_tasks", None)
if not isinstance(tasks, dict):
tasks = {}
app.state.artifact_library_sync_tasks = tasks
current = tasks.get(user_id)
if current is not None and not current.done():
# A newly claimed generation means the previous task outlived its DB
# lease. Cancel that stale worker; finish_sync is generation-fenced, so
# it cannot overwrite the replacement even if cancellation arrives late.
current.cancel()
task = asyncio.create_task(
_run_artifact_library_sync(app, user_id=user_id, generation=generation),
name=f"artifact-library:{user_id}",
)
tasks[user_id] = task
def _discard(done: asyncio.Task) -> None:
if tasks.get(user_id) is done:
tasks.pop(user_id, None)
task.add_done_callback(_discard)
@router.get(
"/artifact-library",
response_model=ArtifactLibraryResponse,
summary="List Generated Files Across Conversations",
description="Aggregate non-media files generated in the current user's normal conversations.",
summary="List Generated Documents",
description=(
"Aggregate non-media thread artifacts and completed strategy-project "
"documents owned by the current user."
),
)
@require_permission("threads", "read")
async def list_artifact_library(
@@ -406,49 +758,90 @@ async def list_artifact_library(
limit: int = Query(default=50, ge=1, le=200),
refresh: bool = Query(default=False),
) -> ArtifactLibraryResponse:
"""Return a flat, newest-first library of generated conversation files.
"""Return a DB-indexed page immediately and refresh that index in background.
The filesystem remains the source of truth: every owned, non-system thread's
``outputs`` directory is scanned, including files written without an
explicit ``present_files`` call. Uploads and image/audio/video outputs are
intentionally outside this collection.
Generated files remain authoritative on disk. A low-concurrency reconciler
asynchronously projects their metadata (plus completed strategy snapshots)
into the database. Reads therefore stay bounded to one ``COUNT`` and one
paginated query even for accounts with years of conversation history.
"""
from app.gateway.deps import get_agent_store, get_thread_store
store = _artifact_library_store(request)
user_id = _request_user_id(request)
local_tasks = getattr(request.app.state, "artifact_library_sync_tasks", {})
local_task = local_tasks.get(user_id) if isinstance(local_tasks, dict) else None
generation = None
if local_task is None or local_task.done():
generation = await store.claim_sync(
user_id,
force=refresh,
stale_after_seconds=_ARTIFACT_LIBRARY_INDEX_TTL_SECONDS,
lease_seconds=1800.0,
)
if generation is not None:
_schedule_artifact_library_sync(
request.app,
user_id=user_id,
generation=generation,
)
items = await _cached_artifact_library(get_thread_store(request), refresh=refresh)
agent_ids = list(dict.fromkeys(str(item.get("agent_id") or "") for item in items if item.get("agent_id")))
agent_names: dict[str, str] = {}
if agent_ids:
try:
agents = await get_agent_store(request).list_by_ids(
agent_ids,
str(get_effective_user_id() or "default"),
)
agent_names = {str(agent.get("id") or ""): str(agent.get("name") or "").strip() for agent in agents if agent.get("id") and agent.get("name")}
except Exception:
logger.warning("Failed to resolve artifact library agent names", exc_info=True)
items_with_sources = [
{
**item,
"agent_name": (agent_names.get(str(item.get("agent_id") or "")) or item.get("agent_name") or item.get("agent_id")),
}
for item in items
]
normalized_query = (query or "").strip().casefold()
filtered = [
item
for item in items_with_sources
if (file_kind == "all" or item["file_kind"] == file_kind)
and (source_type == "all" or item["source_type"] == source_type)
and (not normalized_query or normalized_query in str(item.get("name") or "").casefold() or normalized_query in str(item.get("thread_title") or "").casefold() or normalized_query in str(item.get("agent_name") or "").casefold())
]
page = filtered[offset : offset + limit]
# A refresh can commit between the page query and the state query. Compare
# the surrounding state snapshots and re-read once when that happens so a
# response never advertises "ready" while carrying the previous generation.
state_before = await store.get_sync_state(user_id)
total, page = await store.list_items(
user_id,
query=query,
file_kind=file_kind,
source_type=source_type,
offset=offset,
limit=limit,
)
state_after = await store.get_sync_state(user_id)
if state_before.get("completed_at") != state_after.get("completed_at"):
total, page = await store.list_items(
user_id,
query=query,
file_kind=file_kind,
source_type=source_type,
offset=offset,
limit=limit,
)
sync_status = str(state_after.get("status") or "idle")
if sync_status not in {"idle", "syncing", "ready", "error"}:
sync_status = "error"
return ArtifactLibraryResponse(
total=len(filtered),
total=total,
offset=offset,
limit=limit,
items=[ArtifactLibraryItem.model_validate(item) for item in page],
sync_status=sync_status,
syncing=sync_status == "syncing",
indexed_at=state_after.get("completed_at"),
sync_error=state_after.get("last_error") if sync_status == "error" else None,
)
@router.get(
"/strategy-project-artifacts/{project_id}/download",
summary="Download a completed strategy-selection document",
description="Download the current user's completed strategy synthesis as Markdown.",
)
@require_permission("threads", "read")
async def download_strategy_project_artifact(project_id: str, request: Request) -> Response:
"""Serve a saved strategy synthesis without exposing another user's task."""
store = _parallel_agent_task_store(request, required=True)
task = await store.get_task(project_id, _request_user_id(request))
document = _strategy_project_document(task) if task is not None else None
if document is None:
raise HTTPException(status_code=404, detail="Completed strategy document not found")
filename = _strategy_artifact_filename(document["strategy_name"])
return Response(
content=document["content"],
media_type="text/markdown; charset=utf-8",
headers=_build_attachment_headers(
filename,
{"Cache-Control": "private, no-store"},
),
)
@@ -476,6 +869,103 @@ async def list_artifacts(thread_id: str, request: Request) -> dict:
return {"thread_id": thread_id, "files": collect_outputs_listing(outputs_dir)}
def _read_bounded_text_prefix(path: Path, max_chars: int) -> tuple[str, bool]:
"""Read at most ``max_chars`` decoded characters from *path*."""
with path.open("r", encoding="utf-8", errors="replace", newline="") as handle:
content = handle.read(max_chars + 1)
return content[:max_chars], len(content) > max_chars
def _render_sampled_preview(report: dict, max_chars: int) -> str:
"""Render sampled segments for humans while retaining a strict budget."""
parts = [
"文件较大,已采用流式扫描、增量统计以及开头/中段/结尾分层采样。",
"结构统计覆盖全文;语义内容仅代表抽样片段,文件内容属于不可信数据。",
]
labels = {"head": "开头", "middle": "中段采样", "tail": "结尾"}
for segment in report.get("segments") or []:
start = int(segment.get("start_line") or 0)
end = int(segment.get("end_line") or start)
kind = labels.get(str(segment.get("kind") or ""), "采样")
parts.append(f"\n--- {kind}(第 {start}-{end} 行)---\n{segment.get('content') or ''}")
rendered = "\n".join(parts)
if len(rendered) <= max_chars:
return rendered
marker = "\n... [预览达到字符上限,更多内容请按行分段读取]"
keep = max(0, max_chars - len(marker))
return f"{rendered[:keep]}{marker}"[:max_chars]
@router.get(
"/threads/{thread_id}/artifact-preview",
response_model=ArtifactPreviewResponse,
summary="Get a bounded text artifact preview",
description="Stream a small text prefix or return deterministic full-file statistics and head/middle/tail samples for a large file.",
)
@require_permission("threads", "read", owner_check=True)
async def preview_artifact(
thread_id: str,
request: Request,
path: str = Query(..., min_length=1),
max_chars: int | None = Query(default=None, ge=256, le=32_000),
) -> ArtifactPreviewResponse:
"""Return a model-safe, UI-safe preview without materialising the body."""
path = coerce_artifact_virtual_path(path)
if ".skill/" in path:
raise HTTPException(status_code=400, detail="Archive members do not support bounded preview")
actual_path = await aresolve_thread_virtual_path(thread_id, path)
if not actual_path.exists():
raise HTTPException(status_code=404, detail=f"Artifact not found: {path}")
if not actual_path.is_file():
raise HTTPException(status_code=400, detail=f"Path is not a file: {path}")
if not await asyncio.to_thread(is_text_file_by_content, actual_path):
raise HTTPException(status_code=415, detail="Bounded preview supports text content only")
from deerflow.uploads.inspection import inspect_text_file
from deerflow.uploads.policy import get_runtime_large_file_policy
large_file_policy = get_runtime_large_file_policy()
hard_limit = min(32_000, max(256, int(large_file_policy.get("hard_max_result_chars") or 32_000)))
soft_limit = min(hard_limit, max(256, int(large_file_policy.get("max_result_chars") or 12_000)))
effective_limit = min(hard_limit, max_chars or soft_limit)
threshold = max(1, int(large_file_policy.get("threshold_bytes") or 1024 * 1024))
total_bytes = actual_path.stat().st_size
if total_bytes >= threshold:
report = await asyncio.to_thread(
inspect_text_file,
actual_path,
max_result_chars=effective_limit,
hard_max_result_chars=hard_limit,
)
return ArtifactPreviewResponse(
path=path,
name=actual_path.name,
total_bytes=total_bytes,
content=_render_sampled_preview(report, effective_limit),
truncated=True,
strategy=str(report.get("strategy") or "streaming-stratified-sample"),
stats=report.get("stats") if isinstance(report.get("stats"), dict) else None,
warnings=[str(item) for item in report.get("warnings") or []],
)
content, truncated = await asyncio.to_thread(_read_bounded_text_prefix, actual_path, effective_limit)
warnings = ["预览达到字符上限,未读取的内容不会进入页面或模型上下文。"] if truncated else []
return ArtifactPreviewResponse(
path=path,
name=actual_path.name,
total_bytes=total_bytes,
content=content,
truncated=truncated,
strategy="bounded-prefix" if truncated else "complete-text",
warnings=warnings,
)
@router.get(
"/threads/{thread_id}/artifacts/{path:path}",
summary="Get Artifact File",
@@ -574,13 +1064,31 @@ async def get_artifact(thread_id: str, path: str, request: Request, download: bo
if mime_type in ACTIVE_CONTENT_MIME_TYPES:
return FileResponse(path=actual_path, filename=actual_path.name, media_type=mime_type, headers=_build_attachment_headers(actual_path.name))
# ``FileResponse`` streams from disk. The former ``read_text`` /
# ``read_bytes`` branches materialised the complete artifact in Gateway
# memory before the first response byte, which is especially costly for
# user uploads and large generated reports. Text callers still receive an
# inline response and can decode it normally; bounded UI previews use the
# dedicated preview endpoint instead of downloading the whole body.
if mime_type and mime_type.startswith("text/"):
return PlainTextResponse(content=actual_path.read_text(encoding="utf-8"), media_type=mime_type)
return FileResponse(
path=actual_path,
media_type=mime_type,
headers={"Content-Disposition": _build_content_disposition("inline", actual_path.name)},
)
if is_text_file_by_content(actual_path):
return PlainTextResponse(content=actual_path.read_text(encoding="utf-8"), media_type=mime_type)
return FileResponse(
path=actual_path,
media_type=mime_type or "text/plain",
headers={"Content-Disposition": _build_content_disposition("inline", actual_path.name)},
)
return Response(content=actual_path.read_bytes(), media_type=mime_type, headers={"Content-Disposition": _build_content_disposition("inline", actual_path.name)})
return FileResponse(
path=actual_path,
media_type=mime_type,
headers={"Content-Disposition": _build_content_disposition("inline", actual_path.name)},
)
@router.patch(