From 24a6d83ea004c1694da108476b2c5b6b36ce1ef9 Mon Sep 17 00:00:00 2001 From: SRHTNIN Date: Mon, 28 Sep 2026 23:16:11 +0200 Subject: [PATCH] Add chunked upload for large files --- Backend/Explorer.py | 55 ++++++++++++++++++++-- Backend/Main.py | 45 ++++++++++++++++++ Frontend/src/Api.js | 57 +++++++++++++++++++++++ Frontend/src/Components/DirectoryPane.jsx | 38 +++++++++++---- Frontend/src/Components/Sidebar.jsx | 36 +++++++++++--- 5 files changed, 212 insertions(+), 19 deletions(-) diff --git a/Backend/Explorer.py b/Backend/Explorer.py index 60610f7..037e7a1 100644 --- a/Backend/Explorer.py +++ b/Backend/Explorer.py @@ -1,6 +1,7 @@ from __future__ import annotations import shutil +import tempfile from pathlib import Path from typing import Any @@ -8,6 +9,9 @@ from typing import Any from Core import GetRoots, IsInternalPath, ResolvePath +UploadTempRoot = Path(tempfile.gettempdir()) / "saraview_uploads" + + FileTypeMap = { ".md": "markdown", ".markdown": "markdown", @@ -124,9 +128,7 @@ def MoveEntry(RootName: str, SourceRelativePath: str, DestinationRelativePath: s RenameDirectoryNote(DestinationPath, OldName) -def SaveUploadedFile( - RootName: str, DirectoryRelativePath: str, FileName: str, FileBytes: bytes -) -> str: +def _ResolveUploadTarget(RootName: str, DirectoryRelativePath: str, FileName: str) -> Path: DirectoryPath = ResolvePath(RootName, DirectoryRelativePath) if not DirectoryPath.is_dir(): @@ -142,8 +144,53 @@ def SaveUploadedFile( if TargetPath.exists(): raise FileExistsError(f"{SafeName} already exists.") + return TargetPath + + +def SaveUploadedFile( + RootName: str, DirectoryRelativePath: str, FileName: str, FileBytes: bytes +) -> str: + TargetPath = _ResolveUploadTarget(RootName, DirectoryRelativePath, FileName) TargetPath.write_bytes(FileBytes) - return SafeName + return TargetPath.name + + +def _GetUploadChunkDir(UploadId: str) -> Path: + SafeId = "".join(Char for Char in UploadId if Char.isalnum() or Char in ("-", "_")) + + if not SafeId: + raise ValueError("Invalid upload id.") + + return UploadTempRoot / SafeId + + +def SaveUploadChunk(UploadId: str, ChunkIndex: int, ChunkBytes: bytes) -> None: + ChunkDir = _GetUploadChunkDir(UploadId) + ChunkDir.mkdir(parents=True, exist_ok=True) + (ChunkDir / f"{ChunkIndex:08d}.part").write_bytes(ChunkBytes) + + +def FinishChunkedUpload( + RootName: str, + DirectoryRelativePath: str, + UploadId: str, + TotalChunks: int, + FileName: str, +) -> str: + TargetPath = _ResolveUploadTarget(RootName, DirectoryRelativePath, FileName) + ChunkDir = _GetUploadChunkDir(UploadId) + ChunkPaths = [ChunkDir / f"{Index:08d}.part" for Index in range(TotalChunks)] + + if not all(ChunkPath.exists() for ChunkPath in ChunkPaths): + raise ValueError("Missing upload chunks.") + + with TargetPath.open("wb") as OutputFile: + for ChunkPath in ChunkPaths: + with ChunkPath.open("rb") as ChunkFile: + shutil.copyfileobj(ChunkFile, OutputFile) + + shutil.rmtree(ChunkDir) + return TargetPath.name def DeleteEntry(RootName: str, RelativePath: str) -> None: diff --git a/Backend/Main.py b/Backend/Main.py index d9ff8c2..108a766 100644 --- a/Backend/Main.py +++ b/Backend/Main.py @@ -36,9 +36,11 @@ from Explorer import ( CreateEntry, DeleteEntry, EnsureDirectoryNote, + FinishChunkedUpload, ListDirectory, ListRoots, MoveEntry, + SaveUploadChunk, SaveUploadedFile, ) from Groups import ( @@ -465,6 +467,49 @@ async def PostUploadEndpoint( return {"Saved": SavedNames, "Errors": Errors} +@Application.post("/api/upload/chunk") +async def PostUploadChunkEndpoint( + Root: str, + Path: str, + UploadId: str, + ChunkIndex: int, + TotalChunks: int, + FileName: str, + Chunk: UploadFile = File(...), + CurrentUser: dict = Depends(GetCurrentUser), +) -> dict: + RequirePathAccessible(Root, Path, CurrentUser) + + try: + ChunkBytes = await Chunk.read() + SaveUploadChunk(UploadId, ChunkIndex, ChunkBytes) + except ValueError as Error: + raise HTTPException(status_code=400, detail=str(Error)) + + return {"Received": ChunkIndex} + + +@Application.post("/api/upload/complete") +def PostUploadCompleteEndpoint( + Root: str, + Path: str, + UploadId: str, + TotalChunks: int, + FileName: str, + CurrentUser: dict = Depends(GetCurrentUser), +) -> dict: + RequirePathAccessible(Root, Path, CurrentUser) + + try: + SavedName = FinishChunkedUpload(Root, Path, UploadId, TotalChunks, FileName) + except (ValueError, FileExistsError, NotADirectoryError) as Error: + raise HTTPException(status_code=400, detail=str(Error)) + except OSError as Error: + raise HTTPException(status_code=400, detail=f"Could not save file ({Error})") + + return {"Saved": SavedName} + + class MoveEntryRequest(BaseModel): NewName: str | None = None DestinationPath: str | None = None diff --git a/Frontend/src/Api.js b/Frontend/src/Api.js index 2dafe8f..2048671 100644 --- a/Frontend/src/Api.js +++ b/Frontend/src/Api.js @@ -175,6 +175,63 @@ export async function UploadFiles(RootName, RelativePath, Files) { return ParseJsonOrThrow(Response); } +const UploadChunkSizeBytes = 20 * 1024 * 1024; +export const ChunkedUploadThresholdBytes = 40 * 1024 * 1024; + +export function IsLargeUpload(File) { + return File.size > ChunkedUploadThresholdBytes; +} + +function GenerateUploadId() { + if (window.crypto && window.crypto.randomUUID) { + return window.crypto.randomUUID(); + } + return `${Date.now()}-${Math.random().toString(16).slice(2)}`; +} + +async function UploadFileChunk(RootName, RelativePath, UploadId, ChunkIndex, TotalChunks, FileName, ChunkBlob) { + const Form = new FormData(); + Form.append("Chunk", ChunkBlob, "chunk"); + const Params = new URLSearchParams({ + Root: RootName, + Path: RelativePath, + UploadId, + ChunkIndex: String(ChunkIndex), + TotalChunks: String(TotalChunks), + FileName, + }); + const Response = await fetch(`${ApiBase}/upload/chunk?${Params}`, { + method: "POST", + credentials: "same-origin", + body: Form, + }); + return ParseJsonOrThrow(Response); +} + +function CompleteChunkedUpload(RootName, RelativePath, UploadId, TotalChunks, FileName) { + const Params = new URLSearchParams({ + Root: RootName, + Path: RelativePath, + UploadId, + TotalChunks: String(TotalChunks), + FileName, + }); + return FetchJson(`/upload/complete?${Params}`, { method: "POST" }); +} + +export async function UploadFileChunked(RootName, RelativePath, File) { + const UploadId = GenerateUploadId(); + const TotalChunks = Math.max(1, Math.ceil(File.size / UploadChunkSizeBytes)); + + for (let ChunkIndex = 0; ChunkIndex < TotalChunks; ChunkIndex++) { + const Start = ChunkIndex * UploadChunkSizeBytes; + const ChunkBlob = File.slice(Start, Start + UploadChunkSizeBytes); + await UploadFileChunk(RootName, RelativePath, UploadId, ChunkIndex, TotalChunks, File.name, ChunkBlob); + } + + return CompleteChunkedUpload(RootName, RelativePath, UploadId, TotalChunks, File.name); +} + export function SaveFile(RootName, RelativePath, Content) { const Params = new URLSearchParams({ Root: RootName, Path: RelativePath }); return FetchJson(`/file?${Params}`, { method: "PUT", body: Content }); diff --git a/Frontend/src/Components/DirectoryPane.jsx b/Frontend/src/Components/DirectoryPane.jsx index fa04113..05ed63c 100644 --- a/Frontend/src/Components/DirectoryPane.jsx +++ b/Frontend/src/Components/DirectoryPane.jsx @@ -6,9 +6,11 @@ import { FetchLock, FetchTree, GetDownloadUrl, + IsLargeUpload, MoveEntry, RenameEntry, SetLock as SetLockApi, + UploadFileChunked, UploadFiles, } from "../Api.js"; import { GetFileIcon, GetFolderIcon } from "../IconStyle.js"; @@ -161,16 +163,36 @@ export default function DirectoryPane({ async function HandleUpload(Files) { if (!Files || Files.length === 0) return; - try { - const Result = await UploadFiles(RootName, DirectoryPath, Files); - if (Result.Errors && Result.Errors.length > 0) { - SetErrorMessage(Result.Errors.join("; ")); + const FileArray = Array.from(Files); + const SmallFiles = FileArray.filter((File) => !IsLargeUpload(File)); + const LargeFiles = FileArray.filter((File) => IsLargeUpload(File)); + const Errors = []; + + if (SmallFiles.length > 0) { + try { + const Result = await UploadFiles(RootName, DirectoryPath, SmallFiles); + if (Result.Errors && Result.Errors.length > 0) { + Errors.push(...Result.Errors); + } + } catch (Error) { + Errors.push(Error.message); } - await Reload(); - OnEntryCreated(); - } catch (Error) { - SetErrorMessage(Error.message); } + + for (const File of LargeFiles) { + try { + await UploadFileChunked(RootName, DirectoryPath, File); + } catch (Error) { + Errors.push(`${File.name}: ${Error.message}`); + } + } + + if (Errors.length > 0) { + SetErrorMessage(Errors.join("; ")); + } + + await Reload(); + OnEntryCreated(); } function HandleUploadButtonClick() { diff --git a/Frontend/src/Components/Sidebar.jsx b/Frontend/src/Components/Sidebar.jsx index fdf8ebf..5652056 100644 --- a/Frontend/src/Components/Sidebar.jsx +++ b/Frontend/src/Components/Sidebar.jsx @@ -7,9 +7,11 @@ import { FetchTree, GetAvatarUrl, GetDownloadUrl, + IsLargeUpload, MoveEntry, RenameEntry, SetLock as SetLockApi, + UploadFileChunked, UploadFiles, } from "../Api.js"; import { GetFileIcon, GetFolderIcon } from "../IconStyle.js"; @@ -362,15 +364,35 @@ export default function Sidebar({ async function HandleUpload(Files) { if (!Files || Files.length === 0) return; - try { - const Result = await UploadFiles(RootName, CurrentDirectoryPath, Files); - if (Result.Errors && Result.Errors.length > 0) { - SetErrorMessage(Result.Errors.join("; ")); + const FileArray = Array.from(Files); + const SmallFiles = FileArray.filter((File) => !IsLargeUpload(File)); + const LargeFiles = FileArray.filter((File) => IsLargeUpload(File)); + const Errors = []; + + if (SmallFiles.length > 0) { + try { + const Result = await UploadFiles(RootName, CurrentDirectoryPath, SmallFiles); + if (Result.Errors && Result.Errors.length > 0) { + Errors.push(...Result.Errors); + } + } catch (Error) { + Errors.push(Error.message); } - OnEntryCreated(); - } catch (Error) { - SetErrorMessage(Error.message); } + + for (const File of LargeFiles) { + try { + await UploadFileChunked(RootName, CurrentDirectoryPath, File); + } catch (Error) { + Errors.push(`${File.name}: ${Error.message}`); + } + } + + if (Errors.length > 0) { + SetErrorMessage(Errors.join("; ")); + } + + OnEntryCreated(); } function HandleUploadButtonClick() {