sync: 鍚屾灞€鍩熺綉鏈€鏂颁唬鐮?(source master @ b69568e4)
This commit is contained in:
1 parent
4f7f07504a
commit
8bc8b0e082
1651 files changed
+278599
-11416
No files matched your search
@@ -1,14 +1,18 @@
|
||||
"""Upload router for handling file uploads."""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import stat
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import stat
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, Depends, File, HTTPException, Request, UploadFile
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.gateway.authz import require_permission
|
||||
from app.gateway.deps import get_config
|
||||
from app.gateway.authz import require_permission
|
||||
from app.gateway.deps import get_config
|
||||
from app.gateway.routers.upload_policy import LargeFilePolicy, UploadGroups, get_effective_upload_policy
|
||||
from deerflow.config.app_config import AppConfig
|
||||
from deerflow.config.paths import get_paths
|
||||
from deerflow.runtime.thread_paths import aresolve_path_user_id
|
||||
@@ -22,37 +26,43 @@ from deerflow.uploads.manager import (
|
||||
get_uploads_dir,
|
||||
list_files_in_dir,
|
||||
normalize_filename,
|
||||
open_upload_file_no_symlink,
|
||||
open_upload_file_no_symlink,
|
||||
upload_artifact_url,
|
||||
upload_virtual_path,
|
||||
)
|
||||
upload_virtual_path,
|
||||
validate_path_traversal,
|
||||
)
|
||||
from deerflow.uploads.policy import classify_upload_filename
|
||||
from deerflow.uploads.validation import UploadContentError, validate_upload_content
|
||||
from deerflow.utils.file_conversion import CONVERTIBLE_EXTENSIONS, convert_file_to_markdown
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/threads/{thread_id}/uploads", tags=["uploads"])
|
||||
|
||||
UPLOAD_CHUNK_SIZE = 8192
|
||||
DEFAULT_MAX_FILES = 10
|
||||
DEFAULT_MAX_FILE_SIZE = 50 * 1024 * 1024
|
||||
DEFAULT_MAX_TOTAL_SIZE = 100 * 1024 * 1024
|
||||
|
||||
|
||||
class UploadResponse(BaseModel):
|
||||
UPLOAD_CHUNK_SIZE = 8192
|
||||
|
||||
|
||||
class UploadResponse(BaseModel):
|
||||
"""Response model for file upload."""
|
||||
|
||||
success: bool
|
||||
files: list[dict[str, str]]
|
||||
message: str
|
||||
skipped_files: list[str] = Field(default_factory=list)
|
||||
message: str
|
||||
skipped_files: list[str] = Field(default_factory=list)
|
||||
policy_revision: int = 0
|
||||
|
||||
|
||||
class UploadLimits(BaseModel):
|
||||
class UploadLimits(BaseModel):
|
||||
"""Application-level upload limits exposed to clients."""
|
||||
|
||||
max_files: int
|
||||
max_file_size: int
|
||||
max_total_size: int
|
||||
max_total_size: int
|
||||
max_files_per_message: int
|
||||
max_total_bytes: int
|
||||
groups: UploadGroups
|
||||
large_file: LargeFilePolicy
|
||||
revision: int = 0
|
||||
|
||||
|
||||
def _make_file_sandbox_writable(file_path: os.PathLike[str] | str) -> None:
|
||||
@@ -85,41 +95,22 @@ def _get_uploads_config_value(app_config: AppConfig, key: str, default: object)
|
||||
return getattr(uploads_cfg, key, default)
|
||||
|
||||
|
||||
def _get_upload_limit(app_config: AppConfig, key: str, default: int, *, legacy_key: str | None = None) -> int:
|
||||
try:
|
||||
value = _get_uploads_config_value(app_config, key, None)
|
||||
if value is None and legacy_key is not None:
|
||||
value = _get_uploads_config_value(app_config, legacy_key, None)
|
||||
if value is None:
|
||||
value = default
|
||||
limit = int(value)
|
||||
if limit <= 0:
|
||||
raise ValueError
|
||||
return limit
|
||||
except Exception:
|
||||
logger.warning("Invalid uploads.%s value; falling back to %d", key, default)
|
||||
return default
|
||||
def _get_upload_limits(policy: dict, revision: int = 0) -> UploadLimits:
|
||||
groups = UploadGroups.model_validate(policy["groups"])
|
||||
max_single = max(group.max_bytes for group in (groups.document, groups.image, groups.spreadsheet))
|
||||
return UploadLimits(
|
||||
max_files=int(policy["max_files_per_message"]),
|
||||
max_file_size=max_single,
|
||||
max_total_size=int(policy["max_total_bytes"]),
|
||||
max_files_per_message=int(policy["max_files_per_message"]),
|
||||
max_total_bytes=int(policy["max_total_bytes"]),
|
||||
groups=groups,
|
||||
large_file=LargeFilePolicy.model_validate(policy["large_file"]),
|
||||
revision=revision,
|
||||
)
|
||||
|
||||
|
||||
def _get_upload_limits(app_config: AppConfig) -> UploadLimits:
|
||||
return UploadLimits(
|
||||
max_files=_get_upload_limit(app_config, "max_files", DEFAULT_MAX_FILES, legacy_key="max_file_count"),
|
||||
max_file_size=_get_upload_limit(app_config, "max_file_size", DEFAULT_MAX_FILE_SIZE, legacy_key="max_single_file_size"),
|
||||
max_total_size=_get_upload_limit(app_config, "max_total_size", DEFAULT_MAX_TOTAL_SIZE),
|
||||
)
|
||||
|
||||
|
||||
def _cleanup_uploaded_paths(paths: list[os.PathLike[str] | str]) -> None:
|
||||
for path in reversed(paths):
|
||||
try:
|
||||
os.unlink(path)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
except Exception:
|
||||
logger.warning("Failed to clean up upload path after rejected request: %s", path, exc_info=True)
|
||||
|
||||
|
||||
async def _write_upload_file_with_limits(
|
||||
async def _write_upload_file_with_limits(
|
||||
file: UploadFile,
|
||||
*,
|
||||
uploads_dir: os.PathLike[str] | str,
|
||||
@@ -148,7 +139,63 @@ async def _write_upload_file_with_limits(
|
||||
raise
|
||||
else:
|
||||
fh.close()
|
||||
return file_path, file_size, total_size
|
||||
return file_path, file_size, total_size
|
||||
|
||||
|
||||
def _commit_staged_files(staging_dir: Path, uploads_dir: Path, names: list[str]) -> list[Path]:
|
||||
"""Commit a validated batch and restore every old file on failure."""
|
||||
|
||||
if len(names) != len(set(names)):
|
||||
raise HTTPException(status_code=409, detail="Upload batch contains colliding filenames")
|
||||
backup_dir = staging_dir / ".backup"
|
||||
backup_dir.mkdir()
|
||||
backups: list[tuple[Path, Path]] = []
|
||||
committed: list[Path] = []
|
||||
try:
|
||||
for index, name in enumerate(names):
|
||||
staged = staging_dir / name
|
||||
destination = uploads_dir / name
|
||||
validate_path_traversal(destination, uploads_dir)
|
||||
if destination.exists() or destination.is_symlink():
|
||||
existing_stat = os.lstat(destination)
|
||||
if not stat.S_ISREG(existing_stat.st_mode):
|
||||
raise UnsafeUploadPathError(f"Upload destination is not a regular file: {name}")
|
||||
backup = backup_dir / f"{index}.bak"
|
||||
os.replace(destination, backup)
|
||||
backups.append((destination, backup))
|
||||
os.replace(staged, destination)
|
||||
committed.append(destination)
|
||||
except Exception:
|
||||
for destination in reversed(committed):
|
||||
try:
|
||||
os.unlink(destination)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
for destination, backup in reversed(backups):
|
||||
if backup.exists():
|
||||
os.replace(backup, destination)
|
||||
raise
|
||||
return committed
|
||||
|
||||
|
||||
def _rollback_committed_files(staging_dir: Path, uploads_dir: Path, names: list[str]) -> None:
|
||||
"""Undo :func:`_commit_staged_files` while its backups still exist."""
|
||||
|
||||
backup_dir = staging_dir / ".backup"
|
||||
errors: list[Exception] = []
|
||||
for index, name in reversed(list(enumerate(names))):
|
||||
destination = uploads_dir / name
|
||||
backup = backup_dir / f"{index}.bak"
|
||||
try:
|
||||
if destination.exists() or destination.is_symlink():
|
||||
os.unlink(destination)
|
||||
if backup.exists():
|
||||
os.replace(backup, destination)
|
||||
except Exception as exc: # best-effort through the entire batch
|
||||
errors.append(exc)
|
||||
logger.exception("Failed to roll back committed upload %s", destination)
|
||||
if errors:
|
||||
raise RuntimeError("Failed to fully roll back upload batch") from errors[0]
|
||||
|
||||
|
||||
def _auto_convert_documents_enabled(app_config: AppConfig) -> bool:
|
||||
@@ -173,127 +220,170 @@ async def upload_files(
|
||||
request: Request,
|
||||
files: list[UploadFile] = File(...),
|
||||
config: AppConfig = Depends(get_config),
|
||||
) -> UploadResponse:
|
||||
"""Upload multiple files to a thread's uploads directory."""
|
||||
if not files:
|
||||
raise HTTPException(status_code=400, detail="No files provided")
|
||||
|
||||
limits = _get_upload_limits(config)
|
||||
if len(files) > limits.max_files:
|
||||
raise HTTPException(status_code=413, detail=f"Too many files: maximum is {limits.max_files}")
|
||||
|
||||
try:
|
||||
uploads_dir = ensure_uploads_dir(thread_id)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
sandbox_uploads = get_paths().sandbox_uploads_dir(thread_id, user_id=await aresolve_path_user_id(thread_id))
|
||||
uploaded_files = []
|
||||
written_paths = []
|
||||
sandbox_sync_targets = []
|
||||
skipped_files = []
|
||||
total_size = 0
|
||||
|
||||
sandbox_provider = get_sandbox_provider()
|
||||
sync_to_sandbox = not _uses_thread_data_mounts(sandbox_provider)
|
||||
sandbox = None
|
||||
if sync_to_sandbox:
|
||||
sandbox_id = sandbox_provider.acquire(thread_id)
|
||||
sandbox = sandbox_provider.get(sandbox_id)
|
||||
if sandbox is None:
|
||||
raise HTTPException(status_code=500, detail="Failed to acquire sandbox")
|
||||
auto_convert_documents = _auto_convert_documents_enabled(config)
|
||||
|
||||
for file in files:
|
||||
if not file.filename:
|
||||
continue
|
||||
|
||||
try:
|
||||
safe_filename = normalize_filename(file.filename)
|
||||
except ValueError:
|
||||
logger.warning(f"Skipping file with unsafe filename: {file.filename!r}")
|
||||
continue
|
||||
|
||||
try:
|
||||
file_path, file_size, total_size = await _write_upload_file_with_limits(
|
||||
file,
|
||||
uploads_dir=uploads_dir,
|
||||
display_filename=safe_filename,
|
||||
max_single_file_size=limits.max_file_size,
|
||||
max_total_size=limits.max_total_size,
|
||||
total_size=total_size,
|
||||
)
|
||||
written_paths.append(file_path)
|
||||
|
||||
virtual_path = upload_virtual_path(safe_filename)
|
||||
|
||||
if sync_to_sandbox:
|
||||
sandbox_sync_targets.append((file_path, virtual_path))
|
||||
|
||||
file_info = {
|
||||
"filename": safe_filename,
|
||||
"size": str(file_size),
|
||||
"path": str(sandbox_uploads / safe_filename),
|
||||
"virtual_path": virtual_path,
|
||||
"artifact_url": upload_artifact_url(thread_id, safe_filename),
|
||||
}
|
||||
|
||||
logger.info(f"Saved file: {safe_filename} ({file_size} bytes) to {file_info['path']}")
|
||||
|
||||
file_ext = file_path.suffix.lower()
|
||||
if auto_convert_documents and file_ext in CONVERTIBLE_EXTENSIONS:
|
||||
md_path = await convert_file_to_markdown(file_path)
|
||||
if md_path:
|
||||
written_paths.append(md_path)
|
||||
md_virtual_path = upload_virtual_path(md_path.name)
|
||||
|
||||
if sync_to_sandbox:
|
||||
sandbox_sync_targets.append((md_path, md_virtual_path))
|
||||
|
||||
file_info["markdown_file"] = md_path.name
|
||||
file_info["markdown_path"] = str(sandbox_uploads / md_path.name)
|
||||
file_info["markdown_virtual_path"] = md_virtual_path
|
||||
file_info["markdown_artifact_url"] = upload_artifact_url(thread_id, md_path.name)
|
||||
|
||||
uploaded_files.append(file_info)
|
||||
|
||||
except HTTPException as e:
|
||||
_cleanup_uploaded_paths(written_paths)
|
||||
raise e
|
||||
except UnsafeUploadPathError as e:
|
||||
logger.warning("Skipping upload with unsafe destination %s: %s", file.filename, e)
|
||||
skipped_files.append(safe_filename)
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to upload {file.filename}: {e}")
|
||||
_cleanup_uploaded_paths(written_paths)
|
||||
raise HTTPException(status_code=500, detail=f"Failed to upload {file.filename}: {str(e)}")
|
||||
|
||||
if sync_to_sandbox:
|
||||
for file_path, virtual_path in sandbox_sync_targets:
|
||||
_make_file_sandbox_writable(file_path)
|
||||
sandbox.update_file(virtual_path, file_path.read_bytes())
|
||||
|
||||
message = f"Successfully uploaded {len(uploaded_files)} file(s)"
|
||||
if skipped_files:
|
||||
message += f"; skipped {len(skipped_files)} unsafe file(s)"
|
||||
|
||||
return UploadResponse(
|
||||
success=not skipped_files,
|
||||
files=uploaded_files,
|
||||
message=message,
|
||||
skipped_files=skipped_files,
|
||||
) -> UploadResponse:
|
||||
"""Upload multiple files to a thread's uploads directory."""
|
||||
if not files:
|
||||
raise HTTPException(status_code=400, detail="No files provided")
|
||||
|
||||
policy, policy_metadata = await get_effective_upload_policy(request)
|
||||
limits = _get_upload_limits(policy, int(policy_metadata.get("revision") or 0))
|
||||
if len(files) > limits.max_files:
|
||||
raise HTTPException(status_code=413, detail=f"Too many files: maximum is {limits.max_files}")
|
||||
|
||||
prepared: list[tuple[UploadFile, str, str, int]] = []
|
||||
claimed_names: set[str] = set()
|
||||
for file in files:
|
||||
if not file.filename:
|
||||
raise HTTPException(status_code=400, detail="Every upload must have a filename")
|
||||
try:
|
||||
safe_filename = normalize_filename(file.filename)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
folded_name = safe_filename.casefold()
|
||||
if folded_name in claimed_names:
|
||||
raise HTTPException(status_code=409, detail=f"Duplicate filename in upload batch: {safe_filename}")
|
||||
claimed_names.add(folded_name)
|
||||
classification = classify_upload_filename(safe_filename, policy)
|
||||
if classification is None:
|
||||
raise HTTPException(status_code=415, detail=f"File type is not allowed: {safe_filename}")
|
||||
category, max_bytes = classification
|
||||
prepared.append((file, safe_filename, category, max_bytes))
|
||||
|
||||
try:
|
||||
uploads_dir = ensure_uploads_dir(thread_id)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
sandbox_uploads = get_paths().sandbox_uploads_dir(thread_id, user_id=await aresolve_path_user_id(thread_id))
|
||||
uploaded_files: list[dict[str, str]] = []
|
||||
sandbox_sync_targets: list[tuple[Path, str]] = []
|
||||
total_size = 0
|
||||
auto_convert_documents = _auto_convert_documents_enabled(config)
|
||||
staging_dir = Path(tempfile.mkdtemp(prefix=".upload-staging-", dir=uploads_dir))
|
||||
records: list[dict] = []
|
||||
committed_names: list[str] | None = None
|
||||
try:
|
||||
for file, safe_filename, category, max_bytes in prepared:
|
||||
staged_path, file_size, total_size = await _write_upload_file_with_limits(
|
||||
file,
|
||||
uploads_dir=staging_dir,
|
||||
display_filename=safe_filename,
|
||||
max_single_file_size=max_bytes,
|
||||
max_total_size=limits.max_total_size,
|
||||
total_size=total_size,
|
||||
)
|
||||
file_ext = staged_path.suffix.lower()
|
||||
try:
|
||||
validate_upload_content(staged_path, file_ext)
|
||||
except UploadContentError as exc:
|
||||
raise HTTPException(status_code=415, detail=f"Invalid file {safe_filename}: {exc}") from exc
|
||||
|
||||
record = {
|
||||
"filename": safe_filename,
|
||||
"size": file_size,
|
||||
"category": category,
|
||||
"detected_type": file_ext.removeprefix("."),
|
||||
}
|
||||
if auto_convert_documents and file_ext in CONVERTIBLE_EXTENSIONS:
|
||||
md_path = await convert_file_to_markdown(staged_path)
|
||||
if md_path:
|
||||
markdown_folded = md_path.name.casefold()
|
||||
if markdown_folded in claimed_names:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail=f"Generated Markdown filename collides with another upload: {md_path.name}",
|
||||
)
|
||||
claimed_names.add(markdown_folded)
|
||||
record["markdown_file"] = md_path.name
|
||||
records.append(record)
|
||||
|
||||
sandbox_provider = get_sandbox_provider()
|
||||
sync_to_sandbox = not _uses_thread_data_mounts(sandbox_provider)
|
||||
sandbox = None
|
||||
if sync_to_sandbox:
|
||||
sandbox_id = sandbox_provider.acquire(thread_id)
|
||||
sandbox = sandbox_provider.get(sandbox_id)
|
||||
if sandbox is None:
|
||||
raise HTTPException(status_code=500, detail="Failed to acquire sandbox")
|
||||
|
||||
commit_names = [str(record["filename"]) for record in records]
|
||||
commit_names.extend(str(record["markdown_file"]) for record in records if record.get("markdown_file"))
|
||||
_commit_staged_files(staging_dir, uploads_dir, commit_names)
|
||||
committed_names = commit_names
|
||||
|
||||
for record in records:
|
||||
safe_filename = str(record["filename"])
|
||||
final_path = uploads_dir / safe_filename
|
||||
virtual_path = upload_virtual_path(safe_filename)
|
||||
file_info = {
|
||||
"filename": safe_filename,
|
||||
"size": str(record["size"]),
|
||||
"category": str(record["category"]),
|
||||
"detected_type": str(record["detected_type"]),
|
||||
"path": str(sandbox_uploads / safe_filename),
|
||||
"virtual_path": virtual_path,
|
||||
"artifact_url": upload_artifact_url(thread_id, safe_filename),
|
||||
}
|
||||
if sync_to_sandbox:
|
||||
sandbox_sync_targets.append((final_path, virtual_path))
|
||||
markdown_name = record.get("markdown_file")
|
||||
if markdown_name:
|
||||
markdown_name = str(markdown_name)
|
||||
markdown_path = uploads_dir / markdown_name
|
||||
markdown_virtual_path = upload_virtual_path(markdown_name)
|
||||
file_info.update(
|
||||
{
|
||||
"markdown_file": markdown_name,
|
||||
"markdown_path": str(sandbox_uploads / markdown_name),
|
||||
"markdown_virtual_path": markdown_virtual_path,
|
||||
"markdown_artifact_url": upload_artifact_url(thread_id, markdown_name),
|
||||
}
|
||||
)
|
||||
if sync_to_sandbox:
|
||||
sandbox_sync_targets.append((markdown_path, markdown_virtual_path))
|
||||
uploaded_files.append(file_info)
|
||||
logger.info("Saved validated %s upload: %s (%d bytes)", record["category"], safe_filename, record["size"])
|
||||
|
||||
if sync_to_sandbox and sandbox is not None:
|
||||
for file_path, virtual_path in sandbox_sync_targets:
|
||||
_make_file_sandbox_writable(file_path)
|
||||
sandbox.update_file(virtual_path, file_path.read_bytes())
|
||||
except HTTPException:
|
||||
if committed_names is not None:
|
||||
_rollback_committed_files(staging_dir, uploads_dir, committed_names)
|
||||
raise
|
||||
except UnsafeUploadPathError as exc:
|
||||
if committed_names is not None:
|
||||
_rollback_committed_files(staging_dir, uploads_dir, committed_names)
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
except Exception as exc:
|
||||
logger.exception("Failed to upload validated batch")
|
||||
if committed_names is not None:
|
||||
try:
|
||||
_rollback_committed_files(staging_dir, uploads_dir, committed_names)
|
||||
except Exception:
|
||||
logger.critical("Upload batch rollback failed", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=f"Failed to upload files: {exc}") from exc
|
||||
finally:
|
||||
shutil.rmtree(staging_dir, ignore_errors=True)
|
||||
|
||||
return UploadResponse(
|
||||
success=True,
|
||||
files=uploaded_files,
|
||||
message=f"Successfully uploaded {len(uploaded_files)} file(s)",
|
||||
skipped_files=[],
|
||||
policy_revision=limits.revision,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/limits", response_model=UploadLimits)
|
||||
@require_permission("threads", "read", owner_check=True)
|
||||
async def get_upload_limits(
|
||||
thread_id: str,
|
||||
request: Request,
|
||||
config: AppConfig = Depends(get_config),
|
||||
) -> UploadLimits:
|
||||
"""Return upload limits used by the gateway for this thread."""
|
||||
return _get_upload_limits(config)
|
||||
async def get_upload_limits(
|
||||
thread_id: str,
|
||||
request: Request,
|
||||
) -> UploadLimits:
|
||||
"""Return upload limits used by the gateway for this thread."""
|
||||
policy, metadata = await get_effective_upload_policy(request)
|
||||
return _get_upload_limits(policy, int(metadata.get("revision") or 0))
|
||||
|
||||
|
||||
@router.get("/list", response_model=dict)
|
||||
|
||||
Reference in new issue
Block a user