Add chunked upload for large files

Backend/Explorer.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import shutil
+import tempfile
 
 from pathlib import Path
 from typing import Any
@@ -8,6 +9,9 @@ from typing import Any
 from Core import GetRoots, IsInternalPath, ResolvePath
 
 
+UploadTempRoot = Path(tempfile.gettempdir()) / "saraview_uploads"
+
+
 FileTypeMap = {
     ".md": "markdown",
     ".markdown": "markdown",
@@ -124,9 +128,7 @@ def MoveEntry(RootName: str, SourceRelativePath: str, DestinationRelativePath: s
         RenameDirectoryNote(DestinationPath, OldName)
 
 
-def SaveUploadedFile(
-    RootName: str, DirectoryRelativePath: str, FileName: str, FileBytes: bytes
-) -> str:
+def _ResolveUploadTarget(RootName: str, DirectoryRelativePath: str, FileName: str) -> Path:
     DirectoryPath = ResolvePath(RootName, DirectoryRelativePath)
 
     if not DirectoryPath.is_dir():
@@ -142,8 +144,53 @@ def SaveUploadedFile(
     if TargetPath.exists():
         raise FileExistsError(f"{SafeName} already exists.")
 
+    return TargetPath
+
+
+def SaveUploadedFile(
+    RootName: str, DirectoryRelativePath: str, FileName: str, FileBytes: bytes
+) -> str:
+    TargetPath = _ResolveUploadTarget(RootName, DirectoryRelativePath, FileName)
     TargetPath.write_bytes(FileBytes)
-    return SafeName
+    return TargetPath.name
+
+
+def _GetUploadChunkDir(UploadId: str) -> Path:
+    SafeId = "".join(Char for Char in UploadId if Char.isalnum() or Char in ("-", "_"))
+
+    if not SafeId:
+        raise ValueError("Invalid upload id.")
+
+    return UploadTempRoot / SafeId
+
+
+def SaveUploadChunk(UploadId: str, ChunkIndex: int, ChunkBytes: bytes) -> None:
+    ChunkDir = _GetUploadChunkDir(UploadId)
+    ChunkDir.mkdir(parents=True, exist_ok=True)
+    (ChunkDir / f"{ChunkIndex:08d}.part").write_bytes(ChunkBytes)
+
+
+def FinishChunkedUpload(
+    RootName: str,
+    DirectoryRelativePath: str,
+    UploadId: str,
+    TotalChunks: int,
+    FileName: str,
+) -> str:
+    TargetPath = _ResolveUploadTarget(RootName, DirectoryRelativePath, FileName)
+    ChunkDir = _GetUploadChunkDir(UploadId)
+    ChunkPaths = [ChunkDir / f"{Index:08d}.part" for Index in range(TotalChunks)]
+
+    if not all(ChunkPath.exists() for ChunkPath in ChunkPaths):
+        raise ValueError("Missing upload chunks.")
+
+    with TargetPath.open("wb") as OutputFile:
+        for ChunkPath in ChunkPaths:
+            with ChunkPath.open("rb") as ChunkFile:
+                shutil.copyfileobj(ChunkFile, OutputFile)
+
+    shutil.rmtree(ChunkDir)
+    return TargetPath.name
 
 
 def DeleteEntry(RootName: str, RelativePath: str) -> None:

Backend/Main.py

@@ -36,9 +36,11 @@ from Explorer import (
     CreateEntry,
     DeleteEntry,
     EnsureDirectoryNote,
+    FinishChunkedUpload,
     ListDirectory,
     ListRoots,
     MoveEntry,
+    SaveUploadChunk,
     SaveUploadedFile,
 )
 from Groups import (
@@ -465,6 +467,49 @@ async def PostUploadEndpoint(
     return {"Saved": SavedNames, "Errors": Errors}
 
 
+@Application.post("/api/upload/chunk")
+async def PostUploadChunkEndpoint(
+    Root: str,
+    Path: str,
+    UploadId: str,
+    ChunkIndex: int,
+    TotalChunks: int,
+    FileName: str,
+    Chunk: UploadFile = File(...),
+    CurrentUser: dict = Depends(GetCurrentUser),
+) -> dict:
+    RequirePathAccessible(Root, Path, CurrentUser)
+
+    try:
+        ChunkBytes = await Chunk.read()
+        SaveUploadChunk(UploadId, ChunkIndex, ChunkBytes)
+    except ValueError as Error:
+        raise HTTPException(status_code=400, detail=str(Error))
+
+    return {"Received": ChunkIndex}
+
+
+@Application.post("/api/upload/complete")
+def PostUploadCompleteEndpoint(
+    Root: str,
+    Path: str,
+    UploadId: str,
+    TotalChunks: int,
+    FileName: str,
+    CurrentUser: dict = Depends(GetCurrentUser),
+) -> dict:
+    RequirePathAccessible(Root, Path, CurrentUser)
+
+    try:
+        SavedName = FinishChunkedUpload(Root, Path, UploadId, TotalChunks, FileName)
+    except (ValueError, FileExistsError, NotADirectoryError) as Error:
+        raise HTTPException(status_code=400, detail=str(Error))
+    except OSError as Error:
+        raise HTTPException(status_code=400, detail=f"Could not save file ({Error})")
+
+    return {"Saved": SavedName}
+
+
 class MoveEntryRequest(BaseModel):
     NewName: str | None = None
     DestinationPath: str | None = None

Frontend/src/Api.js

@@ -175,6 +175,63 @@ export async function UploadFiles(RootName, RelativePath, Files) {
   return ParseJsonOrThrow(Response);
 }
 
+const UploadChunkSizeBytes = 20 * 1024 * 1024;
+export const ChunkedUploadThresholdBytes = 40 * 1024 * 1024;
+
+export function IsLargeUpload(File) {
+  return File.size > ChunkedUploadThresholdBytes;
+}
+
+function GenerateUploadId() {
+  if (window.crypto && window.crypto.randomUUID) {
+    return window.crypto.randomUUID();
+  }
+  return `${Date.now()}-${Math.random().toString(16).slice(2)}`;
+}
+
+async function UploadFileChunk(RootName, RelativePath, UploadId, ChunkIndex, TotalChunks, FileName, ChunkBlob) {
+  const Form = new FormData();
+  Form.append("Chunk", ChunkBlob, "chunk");
+  const Params = new URLSearchParams({
+    Root: RootName,
+    Path: RelativePath,
+    UploadId,
+    ChunkIndex: String(ChunkIndex),
+    TotalChunks: String(TotalChunks),
+    FileName,
+  });
+  const Response = await fetch(`${ApiBase}/upload/chunk?${Params}`, {
+    method: "POST",
+    credentials: "same-origin",
+    body: Form,
+  });
+  return ParseJsonOrThrow(Response);
+}
+
+function CompleteChunkedUpload(RootName, RelativePath, UploadId, TotalChunks, FileName) {
+  const Params = new URLSearchParams({
+    Root: RootName,
+    Path: RelativePath,
+    UploadId,
+    TotalChunks: String(TotalChunks),
+    FileName,
+  });
+  return FetchJson(`/upload/complete?${Params}`, { method: "POST" });
+}
+
+export async function UploadFileChunked(RootName, RelativePath, File) {
+  const UploadId = GenerateUploadId();
+  const TotalChunks = Math.max(1, Math.ceil(File.size / UploadChunkSizeBytes));
+
+  for (let ChunkIndex = 0; ChunkIndex < TotalChunks; ChunkIndex++) {
+    const Start = ChunkIndex * UploadChunkSizeBytes;
+    const ChunkBlob = File.slice(Start, Start + UploadChunkSizeBytes);
+    await UploadFileChunk(RootName, RelativePath, UploadId, ChunkIndex, TotalChunks, File.name, ChunkBlob);
+  }
+
+  return CompleteChunkedUpload(RootName, RelativePath, UploadId, TotalChunks, File.name);
+}
+
 export function SaveFile(RootName, RelativePath, Content) {
   const Params = new URLSearchParams({ Root: RootName, Path: RelativePath });
   return FetchJson(`/file?${Params}`, { method: "PUT", body: Content });

Frontend/src/Components/DirectoryPane.jsx

@@ -6,9 +6,11 @@ import {
   FetchLock,
   FetchTree,
   GetDownloadUrl,
+  IsLargeUpload,
   MoveEntry,
   RenameEntry,
   SetLock as SetLockApi,
+  UploadFileChunked,
   UploadFiles,
 } from "../Api.js";
 import { GetFileIcon, GetFolderIcon } from "../IconStyle.js";
@@ -161,16 +163,36 @@ export default function DirectoryPane({
   async function HandleUpload(Files) {
     if (!Files || Files.length === 0) return;
 
-    try {
-      const Result = await UploadFiles(RootName, DirectoryPath, Files);
-      if (Result.Errors && Result.Errors.length > 0) {
-        SetErrorMessage(Result.Errors.join("; "));
+    const FileArray = Array.from(Files);
+    const SmallFiles = FileArray.filter((File) => !IsLargeUpload(File));
+    const LargeFiles = FileArray.filter((File) => IsLargeUpload(File));
+    const Errors = [];
+
+    if (SmallFiles.length > 0) {
+      try {
+        const Result = await UploadFiles(RootName, DirectoryPath, SmallFiles);
+        if (Result.Errors && Result.Errors.length > 0) {
+          Errors.push(...Result.Errors);
+        }
+      } catch (Error) {
+        Errors.push(Error.message);
       }
-      await Reload();
-      OnEntryCreated();
-    } catch (Error) {
-      SetErrorMessage(Error.message);
     }
+
+    for (const File of LargeFiles) {
+      try {
+        await UploadFileChunked(RootName, DirectoryPath, File);
+      } catch (Error) {
+        Errors.push(`${File.name}: ${Error.message}`);
+      }
+    }
+
+    if (Errors.length > 0) {
+      SetErrorMessage(Errors.join("; "));
+    }
+
+    await Reload();
+    OnEntryCreated();
   }
 
   function HandleUploadButtonClick() {

Frontend/src/Components/Sidebar.jsx

@@ -7,9 +7,11 @@ import {
   FetchTree,
   GetAvatarUrl,
   GetDownloadUrl,
+  IsLargeUpload,
   MoveEntry,
   RenameEntry,
   SetLock as SetLockApi,
+  UploadFileChunked,
   UploadFiles,
 } from "../Api.js";
 import { GetFileIcon, GetFolderIcon } from "../IconStyle.js";
@@ -362,15 +364,35 @@ export default function Sidebar({
   async function HandleUpload(Files) {
     if (!Files || Files.length === 0) return;
 
-    try {
-      const Result = await UploadFiles(RootName, CurrentDirectoryPath, Files);
-      if (Result.Errors && Result.Errors.length > 0) {
-        SetErrorMessage(Result.Errors.join("; "));
+    const FileArray = Array.from(Files);
+    const SmallFiles = FileArray.filter((File) => !IsLargeUpload(File));
+    const LargeFiles = FileArray.filter((File) => IsLargeUpload(File));
+    const Errors = [];
+
+    if (SmallFiles.length > 0) {
+      try {
+        const Result = await UploadFiles(RootName, CurrentDirectoryPath, SmallFiles);
+        if (Result.Errors && Result.Errors.length > 0) {
+          Errors.push(...Result.Errors);
+        }
+      } catch (Error) {
+        Errors.push(Error.message);
       }
-      OnEntryCreated();
-    } catch (Error) {
-      SetErrorMessage(Error.message);
     }
+
+    for (const File of LargeFiles) {
+      try {
+        await UploadFileChunked(RootName, CurrentDirectoryPath, File);
+      } catch (Error) {
+        Errors.push(`${File.name}: ${Error.message}`);
+      }
+    }
+
+    if (Errors.length > 0) {
+      SetErrorMessage(Errors.join("; "));
+    }
+
+    OnEntryCreated();
   }
 
   function HandleUploadButtonClick() {