sync: 鍚屾灞€鍩熺綉鏈€鏂颁唬鐮?(source master @ b69568e4)

This commit is contained in:
gozeros committed 2026-10-21 14:51:31 +08:00
1 parent 4f7f07504a
commit 8bc8b0e082
1651 files changed
+278599 -11416

No files matched your search

@@ -1,14 +1,18 @@
"""Upload router for handling file uploads."""
import logging
import os
import stat
import logging
import os
import shutil
import stat
import tempfile
from pathlib import Path
from fastapi import APIRouter, Depends, File, HTTPException, Request, UploadFile
from pydantic import BaseModel, Field
from app.gateway.authz import require_permission
from app.gateway.deps import get_config
from app.gateway.authz import require_permission
from app.gateway.deps import get_config
from app.gateway.routers.upload_policy import LargeFilePolicy, UploadGroups, get_effective_upload_policy
from deerflow.config.app_config import AppConfig
from deerflow.config.paths import get_paths
from deerflow.runtime.thread_paths import aresolve_path_user_id
@@ -22,37 +26,43 @@ from deerflow.uploads.manager import (
get_uploads_dir,
list_files_in_dir,
normalize_filename,
open_upload_file_no_symlink,
open_upload_file_no_symlink,
upload_artifact_url,
upload_virtual_path,
)
upload_virtual_path,
validate_path_traversal,
)
from deerflow.uploads.policy import classify_upload_filename
from deerflow.uploads.validation import UploadContentError, validate_upload_content
from deerflow.utils.file_conversion import CONVERTIBLE_EXTENSIONS, convert_file_to_markdown
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/threads/{thread_id}/uploads", tags=["uploads"])
UPLOAD_CHUNK_SIZE = 8192
DEFAULT_MAX_FILES = 10
DEFAULT_MAX_FILE_SIZE = 50 * 1024 * 1024
DEFAULT_MAX_TOTAL_SIZE = 100 * 1024 * 1024
class UploadResponse(BaseModel):
UPLOAD_CHUNK_SIZE = 8192
class UploadResponse(BaseModel):
"""Response model for file upload."""
success: bool
files: list[dict[str, str]]
message: str
skipped_files: list[str] = Field(default_factory=list)
message: str
skipped_files: list[str] = Field(default_factory=list)
policy_revision: int = 0
class UploadLimits(BaseModel):
class UploadLimits(BaseModel):
"""Application-level upload limits exposed to clients."""
max_files: int
max_file_size: int
max_total_size: int
max_total_size: int
max_files_per_message: int
max_total_bytes: int
groups: UploadGroups
large_file: LargeFilePolicy
revision: int = 0
def _make_file_sandbox_writable(file_path: os.PathLike[str] | str) -> None:
@@ -85,41 +95,22 @@ def _get_uploads_config_value(app_config: AppConfig, key: str, default: object)
return getattr(uploads_cfg, key, default)
def _get_upload_limit(app_config: AppConfig, key: str, default: int, *, legacy_key: str | None = None) -> int:
try:
value = _get_uploads_config_value(app_config, key, None)
if value is None and legacy_key is not None:
value = _get_uploads_config_value(app_config, legacy_key, None)
if value is None:
value = default
limit = int(value)
if limit <= 0:
raise ValueError
return limit
except Exception:
logger.warning("Invalid uploads.%s value; falling back to %d", key, default)
return default
def _get_upload_limits(policy: dict, revision: int = 0) -> UploadLimits:
groups = UploadGroups.model_validate(policy["groups"])
max_single = max(group.max_bytes for group in (groups.document, groups.image, groups.spreadsheet))
return UploadLimits(
max_files=int(policy["max_files_per_message"]),
max_file_size=max_single,
max_total_size=int(policy["max_total_bytes"]),
max_files_per_message=int(policy["max_files_per_message"]),
max_total_bytes=int(policy["max_total_bytes"]),
groups=groups,
large_file=LargeFilePolicy.model_validate(policy["large_file"]),
revision=revision,
)
def _get_upload_limits(app_config: AppConfig) -> UploadLimits:
return UploadLimits(
max_files=_get_upload_limit(app_config, "max_files", DEFAULT_MAX_FILES, legacy_key="max_file_count"),
max_file_size=_get_upload_limit(app_config, "max_file_size", DEFAULT_MAX_FILE_SIZE, legacy_key="max_single_file_size"),
max_total_size=_get_upload_limit(app_config, "max_total_size", DEFAULT_MAX_TOTAL_SIZE),
)
def _cleanup_uploaded_paths(paths: list[os.PathLike[str] | str]) -> None:
for path in reversed(paths):
try:
os.unlink(path)
except FileNotFoundError:
pass
except Exception:
logger.warning("Failed to clean up upload path after rejected request: %s", path, exc_info=True)
async def _write_upload_file_with_limits(
async def _write_upload_file_with_limits(
file: UploadFile,
*,
uploads_dir: os.PathLike[str] | str,
@@ -148,7 +139,63 @@ async def _write_upload_file_with_limits(
raise
else:
fh.close()
return file_path, file_size, total_size
return file_path, file_size, total_size
def _commit_staged_files(staging_dir: Path, uploads_dir: Path, names: list[str]) -> list[Path]:
"""Commit a validated batch and restore every old file on failure."""
if len(names) != len(set(names)):
raise HTTPException(status_code=409, detail="Upload batch contains colliding filenames")
backup_dir = staging_dir / ".backup"
backup_dir.mkdir()
backups: list[tuple[Path, Path]] = []
committed: list[Path] = []
try:
for index, name in enumerate(names):
staged = staging_dir / name
destination = uploads_dir / name
validate_path_traversal(destination, uploads_dir)
if destination.exists() or destination.is_symlink():
existing_stat = os.lstat(destination)
if not stat.S_ISREG(existing_stat.st_mode):
raise UnsafeUploadPathError(f"Upload destination is not a regular file: {name}")
backup = backup_dir / f"{index}.bak"
os.replace(destination, backup)
backups.append((destination, backup))
os.replace(staged, destination)
committed.append(destination)
except Exception:
for destination in reversed(committed):
try:
os.unlink(destination)
except FileNotFoundError:
pass
for destination, backup in reversed(backups):
if backup.exists():
os.replace(backup, destination)
raise
return committed
def _rollback_committed_files(staging_dir: Path, uploads_dir: Path, names: list[str]) -> None:
"""Undo :func:`_commit_staged_files` while its backups still exist."""
backup_dir = staging_dir / ".backup"
errors: list[Exception] = []
for index, name in reversed(list(enumerate(names))):
destination = uploads_dir / name
backup = backup_dir / f"{index}.bak"
try:
if destination.exists() or destination.is_symlink():
os.unlink(destination)
if backup.exists():
os.replace(backup, destination)
except Exception as exc: # best-effort through the entire batch
errors.append(exc)
logger.exception("Failed to roll back committed upload %s", destination)
if errors:
raise RuntimeError("Failed to fully roll back upload batch") from errors[0]
def _auto_convert_documents_enabled(app_config: AppConfig) -> bool:
@@ -173,127 +220,170 @@ async def upload_files(
request: Request,
files: list[UploadFile] = File(...),
config: AppConfig = Depends(get_config),
) -> UploadResponse:
"""Upload multiple files to a thread's uploads directory."""
if not files:
raise HTTPException(status_code=400, detail="No files provided")
limits = _get_upload_limits(config)
if len(files) > limits.max_files:
raise HTTPException(status_code=413, detail=f"Too many files: maximum is {limits.max_files}")
try:
uploads_dir = ensure_uploads_dir(thread_id)
except ValueError as e:
raise HTTPException(status_code=400, detail=str(e))
sandbox_uploads = get_paths().sandbox_uploads_dir(thread_id, user_id=await aresolve_path_user_id(thread_id))
uploaded_files = []
written_paths = []
sandbox_sync_targets = []
skipped_files = []
total_size = 0
sandbox_provider = get_sandbox_provider()
sync_to_sandbox = not _uses_thread_data_mounts(sandbox_provider)
sandbox = None
if sync_to_sandbox:
sandbox_id = sandbox_provider.acquire(thread_id)
sandbox = sandbox_provider.get(sandbox_id)
if sandbox is None:
raise HTTPException(status_code=500, detail="Failed to acquire sandbox")
auto_convert_documents = _auto_convert_documents_enabled(config)
for file in files:
if not file.filename:
continue
try:
safe_filename = normalize_filename(file.filename)
except ValueError:
logger.warning(f"Skipping file with unsafe filename: {file.filename!r}")
continue
try:
file_path, file_size, total_size = await _write_upload_file_with_limits(
file,
uploads_dir=uploads_dir,
display_filename=safe_filename,
max_single_file_size=limits.max_file_size,
max_total_size=limits.max_total_size,
total_size=total_size,
)
written_paths.append(file_path)
virtual_path = upload_virtual_path(safe_filename)
if sync_to_sandbox:
sandbox_sync_targets.append((file_path, virtual_path))
file_info = {
"filename": safe_filename,
"size": str(file_size),
"path": str(sandbox_uploads / safe_filename),
"virtual_path": virtual_path,
"artifact_url": upload_artifact_url(thread_id, safe_filename),
}
logger.info(f"Saved file: {safe_filename} ({file_size} bytes) to {file_info['path']}")
file_ext = file_path.suffix.lower()
if auto_convert_documents and file_ext in CONVERTIBLE_EXTENSIONS:
md_path = await convert_file_to_markdown(file_path)
if md_path:
written_paths.append(md_path)
md_virtual_path = upload_virtual_path(md_path.name)
if sync_to_sandbox:
sandbox_sync_targets.append((md_path, md_virtual_path))
file_info["markdown_file"] = md_path.name
file_info["markdown_path"] = str(sandbox_uploads / md_path.name)
file_info["markdown_virtual_path"] = md_virtual_path
file_info["markdown_artifact_url"] = upload_artifact_url(thread_id, md_path.name)
uploaded_files.append(file_info)
except HTTPException as e:
_cleanup_uploaded_paths(written_paths)
raise e
except UnsafeUploadPathError as e:
logger.warning("Skipping upload with unsafe destination %s: %s", file.filename, e)
skipped_files.append(safe_filename)
continue
except Exception as e:
logger.error(f"Failed to upload {file.filename}: {e}")
_cleanup_uploaded_paths(written_paths)
raise HTTPException(status_code=500, detail=f"Failed to upload {file.filename}: {str(e)}")
if sync_to_sandbox:
for file_path, virtual_path in sandbox_sync_targets:
_make_file_sandbox_writable(file_path)
sandbox.update_file(virtual_path, file_path.read_bytes())
message = f"Successfully uploaded {len(uploaded_files)} file(s)"
if skipped_files:
message += f"; skipped {len(skipped_files)} unsafe file(s)"
return UploadResponse(
success=not skipped_files,
files=uploaded_files,
message=message,
skipped_files=skipped_files,
) -> UploadResponse:
"""Upload multiple files to a thread's uploads directory."""
if not files:
raise HTTPException(status_code=400, detail="No files provided")
policy, policy_metadata = await get_effective_upload_policy(request)
limits = _get_upload_limits(policy, int(policy_metadata.get("revision") or 0))
if len(files) > limits.max_files:
raise HTTPException(status_code=413, detail=f"Too many files: maximum is {limits.max_files}")
prepared: list[tuple[UploadFile, str, str, int]] = []
claimed_names: set[str] = set()
for file in files:
if not file.filename:
raise HTTPException(status_code=400, detail="Every upload must have a filename")
try:
safe_filename = normalize_filename(file.filename)
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
folded_name = safe_filename.casefold()
if folded_name in claimed_names:
raise HTTPException(status_code=409, detail=f"Duplicate filename in upload batch: {safe_filename}")
claimed_names.add(folded_name)
classification = classify_upload_filename(safe_filename, policy)
if classification is None:
raise HTTPException(status_code=415, detail=f"File type is not allowed: {safe_filename}")
category, max_bytes = classification
prepared.append((file, safe_filename, category, max_bytes))
try:
uploads_dir = ensure_uploads_dir(thread_id)
except ValueError as e:
raise HTTPException(status_code=400, detail=str(e))
sandbox_uploads = get_paths().sandbox_uploads_dir(thread_id, user_id=await aresolve_path_user_id(thread_id))
uploaded_files: list[dict[str, str]] = []
sandbox_sync_targets: list[tuple[Path, str]] = []
total_size = 0
auto_convert_documents = _auto_convert_documents_enabled(config)
staging_dir = Path(tempfile.mkdtemp(prefix=".upload-staging-", dir=uploads_dir))
records: list[dict] = []
committed_names: list[str] | None = None
try:
for file, safe_filename, category, max_bytes in prepared:
staged_path, file_size, total_size = await _write_upload_file_with_limits(
file,
uploads_dir=staging_dir,
display_filename=safe_filename,
max_single_file_size=max_bytes,
max_total_size=limits.max_total_size,
total_size=total_size,
)
file_ext = staged_path.suffix.lower()
try:
validate_upload_content(staged_path, file_ext)
except UploadContentError as exc:
raise HTTPException(status_code=415, detail=f"Invalid file {safe_filename}: {exc}") from exc
record = {
"filename": safe_filename,
"size": file_size,
"category": category,
"detected_type": file_ext.removeprefix("."),
}
if auto_convert_documents and file_ext in CONVERTIBLE_EXTENSIONS:
md_path = await convert_file_to_markdown(staged_path)
if md_path:
markdown_folded = md_path.name.casefold()
if markdown_folded in claimed_names:
raise HTTPException(
status_code=409,
detail=f"Generated Markdown filename collides with another upload: {md_path.name}",
)
claimed_names.add(markdown_folded)
record["markdown_file"] = md_path.name
records.append(record)
sandbox_provider = get_sandbox_provider()
sync_to_sandbox = not _uses_thread_data_mounts(sandbox_provider)
sandbox = None
if sync_to_sandbox:
sandbox_id = sandbox_provider.acquire(thread_id)
sandbox = sandbox_provider.get(sandbox_id)
if sandbox is None:
raise HTTPException(status_code=500, detail="Failed to acquire sandbox")
commit_names = [str(record["filename"]) for record in records]
commit_names.extend(str(record["markdown_file"]) for record in records if record.get("markdown_file"))
_commit_staged_files(staging_dir, uploads_dir, commit_names)
committed_names = commit_names
for record in records:
safe_filename = str(record["filename"])
final_path = uploads_dir / safe_filename
virtual_path = upload_virtual_path(safe_filename)
file_info = {
"filename": safe_filename,
"size": str(record["size"]),
"category": str(record["category"]),
"detected_type": str(record["detected_type"]),
"path": str(sandbox_uploads / safe_filename),
"virtual_path": virtual_path,
"artifact_url": upload_artifact_url(thread_id, safe_filename),
}
if sync_to_sandbox:
sandbox_sync_targets.append((final_path, virtual_path))
markdown_name = record.get("markdown_file")
if markdown_name:
markdown_name = str(markdown_name)
markdown_path = uploads_dir / markdown_name
markdown_virtual_path = upload_virtual_path(markdown_name)
file_info.update(
{
"markdown_file": markdown_name,
"markdown_path": str(sandbox_uploads / markdown_name),
"markdown_virtual_path": markdown_virtual_path,
"markdown_artifact_url": upload_artifact_url(thread_id, markdown_name),
}
)
if sync_to_sandbox:
sandbox_sync_targets.append((markdown_path, markdown_virtual_path))
uploaded_files.append(file_info)
logger.info("Saved validated %s upload: %s (%d bytes)", record["category"], safe_filename, record["size"])
if sync_to_sandbox and sandbox is not None:
for file_path, virtual_path in sandbox_sync_targets:
_make_file_sandbox_writable(file_path)
sandbox.update_file(virtual_path, file_path.read_bytes())
except HTTPException:
if committed_names is not None:
_rollback_committed_files(staging_dir, uploads_dir, committed_names)
raise
except UnsafeUploadPathError as exc:
if committed_names is not None:
_rollback_committed_files(staging_dir, uploads_dir, committed_names)
raise HTTPException(status_code=400, detail=str(exc)) from exc
except Exception as exc:
logger.exception("Failed to upload validated batch")
if committed_names is not None:
try:
_rollback_committed_files(staging_dir, uploads_dir, committed_names)
except Exception:
logger.critical("Upload batch rollback failed", exc_info=True)
raise HTTPException(status_code=500, detail=f"Failed to upload files: {exc}") from exc
finally:
shutil.rmtree(staging_dir, ignore_errors=True)
return UploadResponse(
success=True,
files=uploaded_files,
message=f"Successfully uploaded {len(uploaded_files)} file(s)",
skipped_files=[],
policy_revision=limits.revision,
)
@router.get("/limits", response_model=UploadLimits)
@require_permission("threads", "read", owner_check=True)
async def get_upload_limits(
thread_id: str,
request: Request,
config: AppConfig = Depends(get_config),
) -> UploadLimits:
"""Return upload limits used by the gateway for this thread."""
return _get_upload_limits(config)
async def get_upload_limits(
thread_id: str,
request: Request,
) -> UploadLimits:
"""Return upload limits used by the gateway for this thread."""
policy, metadata = await get_effective_upload_policy(request)
return _get_upload_limits(policy, int(metadata.get("revision") or 0))
@router.get("/list", response_model=dict)