sync update from LAN repo (source master @ ca48c4ec)

This commit is contained in:
gozeros committed 2026-11-05 10:53:54 +08:00
1 parent 8bc8b0e082
commit ea9dcff9c0
873 files changed
+107866 -14011

No files matched your search

@@ -1,11 +1,14 @@
"""Upload router for handling file uploads."""
import asyncio
import logging
import os
import shutil
import stat
import tempfile
import threading
from pathlib import Path
from weakref import WeakKeyDictionary, WeakValueDictionary
from fastapi import APIRouter, Depends, File, HTTPException, Request, UploadFile
from pydantic import BaseModel, Field
@@ -33,13 +36,46 @@ from deerflow.uploads.manager import (
)
from deerflow.uploads.policy import classify_upload_filename
from deerflow.uploads.validation import UploadContentError, validate_upload_content
from deerflow.utils.file_conversion import CONVERTIBLE_EXTENSIONS, convert_file_to_markdown
from deerflow.utils.file_conversion import CONVERTIBLE_EXTENSIONS, convert_file_to_markdown_sync
from deerflow.utils.file_io import FileIOPool, file_io
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/threads/{thread_id}/uploads", tags=["uploads"])
UPLOAD_CHUNK_SIZE = 8192
UPLOAD_CHUNK_SIZE = 256 * 1024
# Upload work must not fill the default executor used by authentication, or the
# artifact/event pools used by active conversations. Waiting batches use no worker.
_upload_io = FileIOPool(2, 2, "uploads")
_upload_locks: WeakKeyDictionary = WeakKeyDictionary()
def _upload_lock(directory: Path) -> asyncio.Lock:
locks = _upload_locks.setdefault(asyncio.get_running_loop(), WeakValueDictionary())
key = os.path.normcase(str(directory))
lock = locks.get(key)
if lock is None:
lock = asyncio.Lock()
locks[key] = lock
return lock
class _UploadCancelled(Exception):
"""Cooperative stop inside a worker; the request propagates CancelledError."""
def _check_upload_cancelled(cancelled: threading.Event | None) -> None:
if cancelled is not None and cancelled.is_set():
raise _UploadCancelled()
async def _resolve_uploads_path(thread_id: str) -> Path:
owner = await aresolve_path_user_id(thread_id)
try:
return get_paths().sandbox_uploads_dir(thread_id, user_id=owner)
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
class UploadResponse(BaseModel):
@@ -119,10 +155,37 @@ async def _write_upload_file_with_limits(
max_total_size: int,
total_size: int,
) -> tuple[os.PathLike[str] | str, int, int]:
cancelled = threading.Event()
return await _upload_io.run_until_complete(
_copy_upload_file_with_limits,
file,
uploads_dir=uploads_dir,
display_filename=display_filename,
max_single_file_size=max_single_file_size,
max_total_size=max_total_size,
total_size=total_size,
cancelled=cancelled,
on_cancel=cancelled.set,
)
def _copy_upload_file_with_limits(
file: UploadFile,
*,
uploads_dir: os.PathLike[str] | str,
display_filename: str,
max_single_file_size: int,
max_total_size: int,
total_size: int,
cancelled: threading.Event | None = None,
) -> tuple[Path, int, int]:
"""Copy the parsed multipart spool entirely in the dedicated upload worker."""
_check_upload_cancelled(cancelled)
file_size = 0
file_path, fh = open_upload_file_no_symlink(uploads_dir, display_filename)
try:
while chunk := await file.read(UPLOAD_CHUNK_SIZE):
while chunk := file.file.read(UPLOAD_CHUNK_SIZE):
_check_upload_cancelled(cancelled)
file_size += len(chunk)
total_size += len(chunk)
if file_size > max_single_file_size:
@@ -130,6 +193,7 @@ async def _write_upload_file_with_limits(
if total_size > max_total_size:
raise HTTPException(status_code=413, detail="Total upload size too large")
fh.write(chunk)
_check_upload_cancelled(cancelled)
except Exception:
fh.close()
try:
@@ -249,27 +313,56 @@ async def upload_files(
category, max_bytes = classification
prepared.append((file, safe_filename, category, max_bytes))
# Resolve the canonical owner asynchronously before any filesystem work.
sandbox_uploads = await _resolve_uploads_path(thread_id)
cancelled = threading.Event()
# Moving work off the event loop introduces real concurrency. Serialize
# batches for the same directory so rollback cannot undo another upload.
async with _upload_lock(sandbox_uploads):
return await _upload_io.run_until_complete(
_process_upload_batch,
thread_id,
sandbox_uploads,
prepared,
claimed_names,
limits,
_auto_convert_documents_enabled(config),
cancelled,
on_cancel=cancelled.set,
)
def _process_upload_batch(
thread_id: str,
sandbox_uploads: Path,
prepared: list[tuple[UploadFile, str, str, int]],
claimed_names: set[str],
limits: UploadLimits,
auto_convert_documents: bool,
cancelled: threading.Event,
) -> UploadResponse:
"""Own the full staging/commit/rollback lifetime in one bounded worker."""
_check_upload_cancelled(cancelled)
try:
uploads_dir = ensure_uploads_dir(thread_id)
except ValueError as e:
raise HTTPException(status_code=400, detail=str(e))
sandbox_uploads = get_paths().sandbox_uploads_dir(thread_id, user_id=await aresolve_path_user_id(thread_id))
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
uploaded_files: list[dict[str, str]] = []
sandbox_sync_targets: list[tuple[Path, str]] = []
total_size = 0
auto_convert_documents = _auto_convert_documents_enabled(config)
staging_dir = Path(tempfile.mkdtemp(prefix=".upload-staging-", dir=uploads_dir))
records: list[dict] = []
committed_names: list[str] | None = None
try:
for file, safe_filename, category, max_bytes in prepared:
staged_path, file_size, total_size = await _write_upload_file_with_limits(
staged_path, file_size, total_size = _copy_upload_file_with_limits(
file,
uploads_dir=staging_dir,
display_filename=safe_filename,
max_single_file_size=max_bytes,
max_total_size=limits.max_total_size,
total_size=total_size,
cancelled=cancelled,
)
file_ext = staged_path.suffix.lower()
try:
@@ -284,7 +377,8 @@ async def upload_files(
"detected_type": file_ext.removeprefix("."),
}
if auto_convert_documents and file_ext in CONVERTIBLE_EXTENSIONS:
md_path = await convert_file_to_markdown(staged_path)
_check_upload_cancelled(cancelled)
md_path = convert_file_to_markdown_sync(staged_path)
if md_path:
markdown_folded = md_path.name.casefold()
if markdown_folded in claimed_names:
@@ -296,6 +390,7 @@ async def upload_files(
record["markdown_file"] = md_path.name
records.append(record)
_check_upload_cancelled(cancelled)
sandbox_provider = get_sandbox_provider()
sync_to_sandbox = not _uses_thread_data_mounts(sandbox_provider)
sandbox = None
@@ -307,6 +402,7 @@ async def upload_files(
commit_names = [str(record["filename"]) for record in records]
commit_names.extend(str(record["markdown_file"]) for record in records if record.get("markdown_file"))
_check_upload_cancelled(cancelled)
_commit_staged_files(staging_dir, uploads_dir, commit_names)
committed_names = commit_names
@@ -345,9 +441,11 @@ async def upload_files(
if sync_to_sandbox and sandbox is not None:
for file_path, virtual_path in sandbox_sync_targets:
_check_upload_cancelled(cancelled)
_make_file_sandbox_writable(file_path)
sandbox.update_file(virtual_path, file_path.read_bytes())
except HTTPException:
_check_upload_cancelled(cancelled)
except (HTTPException, _UploadCancelled):
if committed_names is not None:
_rollback_committed_files(staging_dir, uploads_dir, committed_names)
raise
@@ -390,6 +488,12 @@ async def get_upload_limits(
@require_permission("threads", "read", owner_check=True)
async def list_uploaded_files(thread_id: str, request: Request) -> dict:
"""List all files in a thread's uploads directory."""
sandbox_uploads = await _resolve_uploads_path(thread_id)
async with _upload_lock(sandbox_uploads):
return await file_io.run_until_complete(_list_uploaded_files, thread_id, sandbox_uploads)
def _list_uploaded_files(thread_id: str, sandbox_uploads: Path) -> dict:
try:
uploads_dir = get_uploads_dir(thread_id)
except ValueError as e:
@@ -398,7 +502,6 @@ async def list_uploaded_files(thread_id: str, request: Request) -> dict:
enrich_file_listing(result, thread_id)
# Gateway additionally includes the sandbox-relative path.
sandbox_uploads = get_paths().sandbox_uploads_dir(thread_id, user_id=await aresolve_path_user_id(thread_id))
for f in result["files"]:
f["path"] = str(sandbox_uploads / f["filename"])
@@ -409,6 +512,12 @@ async def list_uploaded_files(thread_id: str, request: Request) -> dict:
@require_permission("threads", "delete", owner_check=True, require_existing=True)
async def delete_uploaded_file(thread_id: str, filename: str, request: Request) -> dict:
"""Delete a file from a thread's uploads directory."""
sandbox_uploads = await _resolve_uploads_path(thread_id)
async with _upload_lock(sandbox_uploads):
return await file_io.run_until_complete(_delete_uploaded_file, thread_id, filename)
def _delete_uploaded_file(thread_id: str, filename: str) -> dict:
try:
uploads_dir = get_uploads_dir(thread_id)
except ValueError as e: