sync update from LAN repo (source dhk-1 @ 3b1ec5b4)
This commit is contained in:
1 parent
da6521b6cb
commit
264e2200d6
995 files changed
+148105
-13399
No files matched your search
@@ -1,11 +1,14 @@
|
||||
"""Upload router for handling file uploads."""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import stat
|
||||
import tempfile
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from weakref import WeakKeyDictionary, WeakValueDictionary
|
||||
|
||||
from fastapi import APIRouter, Depends, File, HTTPException, Request, UploadFile
|
||||
from pydantic import BaseModel, Field
|
||||
@@ -33,13 +36,46 @@ from deerflow.uploads.manager import (
|
||||
)
|
||||
from deerflow.uploads.policy import classify_upload_filename
|
||||
from deerflow.uploads.validation import UploadContentError, validate_upload_content
|
||||
from deerflow.utils.file_conversion import CONVERTIBLE_EXTENSIONS, convert_file_to_markdown
|
||||
from deerflow.utils.file_conversion import CONVERTIBLE_EXTENSIONS, convert_file_to_markdown_sync
|
||||
from deerflow.utils.file_io import FileIOPool, file_io
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/threads/{thread_id}/uploads", tags=["uploads"])
|
||||
|
||||
UPLOAD_CHUNK_SIZE = 8192
|
||||
UPLOAD_CHUNK_SIZE = 256 * 1024
|
||||
|
||||
# Upload work must not fill the default executor used by authentication, or the
|
||||
# artifact/event pools used by active conversations. Waiting batches use no worker.
|
||||
_upload_io = FileIOPool(2, 2, "uploads")
|
||||
_upload_locks: WeakKeyDictionary = WeakKeyDictionary()
|
||||
|
||||
|
||||
def _upload_lock(directory: Path) -> asyncio.Lock:
|
||||
locks = _upload_locks.setdefault(asyncio.get_running_loop(), WeakValueDictionary())
|
||||
key = os.path.normcase(str(directory))
|
||||
lock = locks.get(key)
|
||||
if lock is None:
|
||||
lock = asyncio.Lock()
|
||||
locks[key] = lock
|
||||
return lock
|
||||
|
||||
|
||||
class _UploadCancelled(Exception):
|
||||
"""Cooperative stop inside a worker; the request propagates CancelledError."""
|
||||
|
||||
|
||||
def _check_upload_cancelled(cancelled: threading.Event | None) -> None:
|
||||
if cancelled is not None and cancelled.is_set():
|
||||
raise _UploadCancelled()
|
||||
|
||||
|
||||
async def _resolve_uploads_path(thread_id: str) -> Path:
|
||||
owner = await aresolve_path_user_id(thread_id)
|
||||
try:
|
||||
return get_paths().sandbox_uploads_dir(thread_id, user_id=owner)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
|
||||
class UploadResponse(BaseModel):
|
||||
@@ -119,10 +155,37 @@ async def _write_upload_file_with_limits(
|
||||
max_total_size: int,
|
||||
total_size: int,
|
||||
) -> tuple[os.PathLike[str] | str, int, int]:
|
||||
cancelled = threading.Event()
|
||||
return await _upload_io.run_until_complete(
|
||||
_copy_upload_file_with_limits,
|
||||
file,
|
||||
uploads_dir=uploads_dir,
|
||||
display_filename=display_filename,
|
||||
max_single_file_size=max_single_file_size,
|
||||
max_total_size=max_total_size,
|
||||
total_size=total_size,
|
||||
cancelled=cancelled,
|
||||
on_cancel=cancelled.set,
|
||||
)
|
||||
|
||||
|
||||
def _copy_upload_file_with_limits(
|
||||
file: UploadFile,
|
||||
*,
|
||||
uploads_dir: os.PathLike[str] | str,
|
||||
display_filename: str,
|
||||
max_single_file_size: int,
|
||||
max_total_size: int,
|
||||
total_size: int,
|
||||
cancelled: threading.Event | None = None,
|
||||
) -> tuple[Path, int, int]:
|
||||
"""Copy the parsed multipart spool entirely in the dedicated upload worker."""
|
||||
_check_upload_cancelled(cancelled)
|
||||
file_size = 0
|
||||
file_path, fh = open_upload_file_no_symlink(uploads_dir, display_filename)
|
||||
try:
|
||||
while chunk := await file.read(UPLOAD_CHUNK_SIZE):
|
||||
while chunk := file.file.read(UPLOAD_CHUNK_SIZE):
|
||||
_check_upload_cancelled(cancelled)
|
||||
file_size += len(chunk)
|
||||
total_size += len(chunk)
|
||||
if file_size > max_single_file_size:
|
||||
@@ -130,6 +193,7 @@ async def _write_upload_file_with_limits(
|
||||
if total_size > max_total_size:
|
||||
raise HTTPException(status_code=413, detail="Total upload size too large")
|
||||
fh.write(chunk)
|
||||
_check_upload_cancelled(cancelled)
|
||||
except Exception:
|
||||
fh.close()
|
||||
try:
|
||||
@@ -249,27 +313,56 @@ async def upload_files(
|
||||
category, max_bytes = classification
|
||||
prepared.append((file, safe_filename, category, max_bytes))
|
||||
|
||||
# Resolve the canonical owner asynchronously before any filesystem work.
|
||||
sandbox_uploads = await _resolve_uploads_path(thread_id)
|
||||
cancelled = threading.Event()
|
||||
# Moving work off the event loop introduces real concurrency. Serialize
|
||||
# batches for the same directory so rollback cannot undo another upload.
|
||||
async with _upload_lock(sandbox_uploads):
|
||||
return await _upload_io.run_until_complete(
|
||||
_process_upload_batch,
|
||||
thread_id,
|
||||
sandbox_uploads,
|
||||
prepared,
|
||||
claimed_names,
|
||||
limits,
|
||||
_auto_convert_documents_enabled(config),
|
||||
cancelled,
|
||||
on_cancel=cancelled.set,
|
||||
)
|
||||
|
||||
|
||||
def _process_upload_batch(
|
||||
thread_id: str,
|
||||
sandbox_uploads: Path,
|
||||
prepared: list[tuple[UploadFile, str, str, int]],
|
||||
claimed_names: set[str],
|
||||
limits: UploadLimits,
|
||||
auto_convert_documents: bool,
|
||||
cancelled: threading.Event,
|
||||
) -> UploadResponse:
|
||||
"""Own the full staging/commit/rollback lifetime in one bounded worker."""
|
||||
_check_upload_cancelled(cancelled)
|
||||
try:
|
||||
uploads_dir = ensure_uploads_dir(thread_id)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
sandbox_uploads = get_paths().sandbox_uploads_dir(thread_id, user_id=await aresolve_path_user_id(thread_id))
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
uploaded_files: list[dict[str, str]] = []
|
||||
sandbox_sync_targets: list[tuple[Path, str]] = []
|
||||
total_size = 0
|
||||
auto_convert_documents = _auto_convert_documents_enabled(config)
|
||||
staging_dir = Path(tempfile.mkdtemp(prefix=".upload-staging-", dir=uploads_dir))
|
||||
records: list[dict] = []
|
||||
committed_names: list[str] | None = None
|
||||
try:
|
||||
for file, safe_filename, category, max_bytes in prepared:
|
||||
staged_path, file_size, total_size = await _write_upload_file_with_limits(
|
||||
staged_path, file_size, total_size = _copy_upload_file_with_limits(
|
||||
file,
|
||||
uploads_dir=staging_dir,
|
||||
display_filename=safe_filename,
|
||||
max_single_file_size=max_bytes,
|
||||
max_total_size=limits.max_total_size,
|
||||
total_size=total_size,
|
||||
cancelled=cancelled,
|
||||
)
|
||||
file_ext = staged_path.suffix.lower()
|
||||
try:
|
||||
@@ -284,7 +377,8 @@ async def upload_files(
|
||||
"detected_type": file_ext.removeprefix("."),
|
||||
}
|
||||
if auto_convert_documents and file_ext in CONVERTIBLE_EXTENSIONS:
|
||||
md_path = await convert_file_to_markdown(staged_path)
|
||||
_check_upload_cancelled(cancelled)
|
||||
md_path = convert_file_to_markdown_sync(staged_path)
|
||||
if md_path:
|
||||
markdown_folded = md_path.name.casefold()
|
||||
if markdown_folded in claimed_names:
|
||||
@@ -296,6 +390,7 @@ async def upload_files(
|
||||
record["markdown_file"] = md_path.name
|
||||
records.append(record)
|
||||
|
||||
_check_upload_cancelled(cancelled)
|
||||
sandbox_provider = get_sandbox_provider()
|
||||
sync_to_sandbox = not _uses_thread_data_mounts(sandbox_provider)
|
||||
sandbox = None
|
||||
@@ -307,6 +402,7 @@ async def upload_files(
|
||||
|
||||
commit_names = [str(record["filename"]) for record in records]
|
||||
commit_names.extend(str(record["markdown_file"]) for record in records if record.get("markdown_file"))
|
||||
_check_upload_cancelled(cancelled)
|
||||
_commit_staged_files(staging_dir, uploads_dir, commit_names)
|
||||
committed_names = commit_names
|
||||
|
||||
@@ -345,9 +441,11 @@ async def upload_files(
|
||||
|
||||
if sync_to_sandbox and sandbox is not None:
|
||||
for file_path, virtual_path in sandbox_sync_targets:
|
||||
_check_upload_cancelled(cancelled)
|
||||
_make_file_sandbox_writable(file_path)
|
||||
sandbox.update_file(virtual_path, file_path.read_bytes())
|
||||
except HTTPException:
|
||||
_check_upload_cancelled(cancelled)
|
||||
except (HTTPException, _UploadCancelled):
|
||||
if committed_names is not None:
|
||||
_rollback_committed_files(staging_dir, uploads_dir, committed_names)
|
||||
raise
|
||||
@@ -390,6 +488,12 @@ async def get_upload_limits(
|
||||
@require_permission("threads", "read", owner_check=True)
|
||||
async def list_uploaded_files(thread_id: str, request: Request) -> dict:
|
||||
"""List all files in a thread's uploads directory."""
|
||||
sandbox_uploads = await _resolve_uploads_path(thread_id)
|
||||
async with _upload_lock(sandbox_uploads):
|
||||
return await file_io.run_until_complete(_list_uploaded_files, thread_id, sandbox_uploads)
|
||||
|
||||
|
||||
def _list_uploaded_files(thread_id: str, sandbox_uploads: Path) -> dict:
|
||||
try:
|
||||
uploads_dir = get_uploads_dir(thread_id)
|
||||
except ValueError as e:
|
||||
@@ -398,7 +502,6 @@ async def list_uploaded_files(thread_id: str, request: Request) -> dict:
|
||||
enrich_file_listing(result, thread_id)
|
||||
|
||||
# Gateway additionally includes the sandbox-relative path.
|
||||
sandbox_uploads = get_paths().sandbox_uploads_dir(thread_id, user_id=await aresolve_path_user_id(thread_id))
|
||||
for f in result["files"]:
|
||||
f["path"] = str(sandbox_uploads / f["filename"])
|
||||
|
||||
@@ -409,6 +512,12 @@ async def list_uploaded_files(thread_id: str, request: Request) -> dict:
|
||||
@require_permission("threads", "delete", owner_check=True, require_existing=True)
|
||||
async def delete_uploaded_file(thread_id: str, filename: str, request: Request) -> dict:
|
||||
"""Delete a file from a thread's uploads directory."""
|
||||
sandbox_uploads = await _resolve_uploads_path(thread_id)
|
||||
async with _upload_lock(sandbox_uploads):
|
||||
return await file_io.run_until_complete(_delete_uploaded_file, thread_id, filename)
|
||||
|
||||
|
||||
def _delete_uploaded_file(thread_id: str, filename: str) -> dict:
|
||||
try:
|
||||
uploads_dir = get_uploads_dir(thread_id)
|
||||
except ValueError as e:
|
||||
|
||||
Reference in new issue
Block a user