Implemented chunked uploading. Updated mooncore

Moonlight.ApiServer/Http/Controllers/Admin/Sys/FilesController.cs

@@ -6,7 +6,6 @@ using Microsoft.AspNetCore.Mvc;
 using MoonCore.Exceptions;
 using MoonCore.Extended.PermFilter;
 using MoonCore.Helpers;
-using Moonlight.ApiServer.Configuration;
 using Moonlight.Shared.Http.Requests.Admin.Sys.Files;
 using Moonlight.Shared.Http.Responses.Admin.Sys;
 
@@ -18,6 +17,7 @@ namespace Moonlight.ApiServer.Http.Controllers.Admin.Sys;
 public class FilesController : Controller
 {
     private readonly string BaseDirectory = PathBuilder.Dir("storage");
+    private readonly long ChunkSize = ByteConverter.FromMegaBytes(20).Bytes;
 
     [HttpGet("list")]
     public Task<FileSystemEntryResponse[]> List([FromQuery] string path)
@@ -64,14 +64,24 @@ public class FilesController : Controller
         );
     }
 
-    [HttpPost("create")]
-    public async Task Create([FromQuery] string path)
+    [HttpPost("upload")]
+    public async Task Upload([FromQuery] string path, [FromQuery] long totalSize, [FromQuery] int chunkId)
     {
         if (Request.Form.Files.Count != 1)
             throw new HttpApiException("You need to provide exactly one file", 400);
 
         var file = Request.Form.Files[0];
-        var stream = file.OpenReadStream();
+        
+        if (file.Length > ChunkSize)
+            throw new HttpApiException("The provided data exceeds the chunk size limit", 400);
+        
+        var chunks = totalSize / ChunkSize;
+        chunks += totalSize % ChunkSize > 0 ? 1 : 0;
+
+        if (chunkId > chunks)
+            throw new HttpApiException("Invalid chunk id: Out of bounds", 400);
+        
+        var positionToSkipTo = ChunkSize * chunkId;
 
         var safePath = SanitizePath(path);
         var physicalPath = PathBuilder.File(BaseDirectory, safePath);
@@ -80,15 +90,22 @@ public class FilesController : Controller
         if (!string.IsNullOrEmpty(baseDir))
             Directory.CreateDirectory(baseDir);
 
-        await using var fs = System.IO.File.Create(
-            physicalPath
-        );
+        var didExistBefore = System.IO.File.Exists(physicalPath);
 
-        await stream.CopyToAsync(fs);
+        await using var fs = System.IO.File.Open(physicalPath, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None);
 
+        // This creates the file in the correct size so we can handle the chunk if it didnt exist
+        if (!didExistBefore)
+            fs.SetLength(totalSize);
+        
+        fs.Position = positionToSkipTo;
+
+        var dataStream = file.OpenReadStream();
+
+        await dataStream.CopyToAsync(fs);
         await fs.FlushAsync();
+
         fs.Close();
-        stream.Close();
     }
 
     [HttpPost("move")]
@@ -150,8 +167,8 @@ public class FilesController : Controller
         return Task.CompletedTask;
     }
 
-    [HttpGet("read")]
-    public async Task Read([FromQuery] string path)
+    [HttpGet("download")]
+    public async Task Download([FromQuery] string path)
     {
         var safePath = SanitizePath(path);
         var physicalPath = PathBuilder.File(BaseDirectory, safePath);

Moonlight.Client/Implementations/SysFileSystemProvider.cs

@@ -2,7 +2,6 @@
 using MoonCore.Blazor.Tailwind.Fm;
 using MoonCore.Blazor.Tailwind.Fm.Models;
 using MoonCore.Blazor.Tailwind.Services;
-using MoonCore.Blazor.Tailwind.Xhr;
 using MoonCore.Helpers;
 using Moonlight.Shared.Http.Requests.Admin.Sys.Files;
 using Moonlight.Shared.Http.Responses.Admin.Sys;
@@ -14,7 +13,6 @@ public class SysFileSystemProvider : IFileSystemProvider, ICompressFileSystemPro
     private readonly DownloadService DownloadService;
     private readonly HttpApiClient HttpApiClient;
     private readonly LocalStorageService LocalStorageService;
-    private readonly XmlHttpClient XmlHttpClient;
     private readonly string BaseApiUrl = "api/admin/system/files";
 
     public CompressType[] CompressTypes { get; } =
@@ -31,12 +29,15 @@ public class SysFileSystemProvider : IFileSystemProvider, ICompressFileSystemPro
         }
     ];
 
-    public SysFileSystemProvider(HttpApiClient httpApiClient, DownloadService downloadService, LocalStorageService localStorageService, XmlHttpClient xmlHttpClient)
+    public SysFileSystemProvider(
+        HttpApiClient httpApiClient,
+        DownloadService downloadService,
+        LocalStorageService localStorageService
+    )
     {
         HttpApiClient = httpApiClient;
         DownloadService = downloadService;
         LocalStorageService = localStorageService;
-        XmlHttpClient = xmlHttpClient;
     }
 
     public async Task<FileSystemEntry[]> List(string path)
@@ -57,11 +58,7 @@ public class SysFileSystemProvider : IFileSystemProvider, ICompressFileSystemPro
 
     public async Task Create(string path, Stream stream)
     {
-        var content = new MultipartFormDataContent();
-        
-        content.Add(new StreamContent(stream), "file", "x");
-        
-        await HttpApiClient.Post($"{BaseApiUrl}/create?path={path}", content);
+        await Upload(_ => Task.CompletedTask, path, stream);
     }
 
     public async Task Move(string oldPath, string newPath)
@@ -74,45 +71,46 @@ public class SysFileSystemProvider : IFileSystemProvider, ICompressFileSystemPro
         => await HttpApiClient.Post($"{BaseApiUrl}/mkdir?path={path}");
 
     public async Task<Stream> Read(string path)
-        => await HttpApiClient.GetStream($"{BaseApiUrl}/read?path={path}");
+        => await HttpApiClient.GetStream($"{BaseApiUrl}/download?path={path}");
 
-    public async Task Download(Func<long, Task> updateProgress, string path, string fileName)
+    public async Task Download(Func<int, Task> updateProgress, string path, string fileName)
     {
         var accessToken = await LocalStorageService.GetString("AccessToken");
 
-        await DownloadService.DownloadUrl(fileName, $"{BaseApiUrl}/read?path={path}", async (bytes, _) =>
-        {
-            await updateProgress.Invoke(bytes);
-        }, headers =>
-        {
-            headers.Add("Authorization", $"Bearer {accessToken}");
-        });
+        await DownloadService.DownloadUrl(fileName, $"{BaseApiUrl}/download?path={path}",
+            async (loaded, total) =>
+            {
+                var percent = total == 0 ? 0 : (int)Math.Round((float)loaded / total * 100);
+                await updateProgress.Invoke(percent);
+            },
+            onConfigureHeaders: headers => { headers.Add("Authorization", $"Bearer {accessToken}"); }
+        );
     }
 
-    public async Task Upload(Func<long, Task> updateProgress, string path, Stream stream)
+    public async Task Upload(Func<int, Task> updateProgress, string path, Stream stream)
     {
-        var tcs = new TaskCompletionSource();
-        var accessToken = await LocalStorageService.GetString("AccessToken");
+        var size = stream.Length;
+        var chunkSize = ByteConverter.FromMegaBytes(20).Bytes;
 
-        await using var request = await XmlHttpClient.Create();
+        var chunks = size / chunkSize;
+        chunks += size % chunkSize > 0 ? 1 : 0;
         
-        request.OnUploadProgress += async ev =>
+        for (var chunkId = 0; chunkId < chunks; chunkId++)
         {
-            await updateProgress.Invoke(ev.Loaded);
-        };
+            var percent = (int)Math.Round((chunkId + 1f) / chunks * 100);
+            await updateProgress.Invoke(percent);
             
-        request.OnLoadend += _ =>
-        {
-            tcs.SetResult();
-            return Task.CompletedTask;
-        };
+            var buffer = new byte[chunkSize];
+            var bytesRead = await stream.ReadAsync(buffer);
 
-        await request.Open("POST", $"{BaseApiUrl}/create?path={path}");
-        await request.SetRequestHeader("Authorization", $"Bearer {accessToken}");
+            var uploadForm = new MultipartFormDataContent();
+            uploadForm.Add(new ByteArrayContent(buffer, 0, bytesRead), "file", path);
 
-        await request.SendFile(stream, "file", "file");
-
-        await tcs.Task;
+            await HttpApiClient.Post(
+                $"{BaseApiUrl}/upload?path={path}&totalSize={size}&chunkId={chunkId}",
+                uploadForm
+            );
+        }
     }
 
     public async Task Compress(CompressType type, string path, string[] itemsToCompress)

Moonlight.Client/Moonlight.Client.csproj

@@ -27,7 +27,7 @@
         <PackageReference Include="MoonCore" Version="1.8.5" />
         <PackageReference Include="MoonCore.Blazor" Version="1.2.9" />
         <PackageReference Include="MoonCore.PluginFramework" Version="1.0.5"/>
-        <PackageReference Include="MoonCore.Blazor.Tailwind" Version="1.3.7" />
+        <PackageReference Include="MoonCore.Blazor.Tailwind" Version="1.3.9" />
     </ItemGroup>
 
     <!-- 

Moonlight.Client/Startup.cs

@@ -7,7 +7,6 @@ using Microsoft.JSInterop;
 using MoonCore.Blazor.Services;
 using MoonCore.Blazor.Tailwind.Extensions;
 using MoonCore.Blazor.Tailwind.Auth;
-using MoonCore.Blazor.Tailwind.Xhr;
 using MoonCore.Extensions;
 using MoonCore.Helpers;
 using Moonlight.Client.Interfaces;
@@ -146,8 +145,6 @@ public class Startup
         WebAssemblyHostBuilder.Services.AddMoonCoreBlazorTailwind();
         WebAssemblyHostBuilder.Services.AddScoped<LocalStorageService>();
 
-        WebAssemblyHostBuilder.Services.AddScoped<XmlHttpClient>();
-
         WebAssemblyHostBuilder.Services.AddScoped<ThemeService>();
         
         WebAssemblyHostBuilder.Services.AutoAddServices<Program>();

Moonlight.Client/UI/Views/Admin/Sys/Files.razor

@@ -4,7 +4,6 @@
 @using MoonCore.Blazor.Services
 @using MoonCore.Helpers
 @using MoonCore.Blazor.Tailwind.Fm
-@using MoonCore.Blazor.Tailwind.Xhr
 @using Moonlight.Client.Implementations
 
 @attribute [RequirePermission("admin.system.overview")]
@@ -12,13 +11,12 @@
 @inject HttpApiClient ApiClient
 @inject DownloadService DownloadService
 @inject LocalStorageService LocalStorageService
-@inject XmlHttpClient XmlHttpClient
 
 <div class="mb-3">
     <NavTabs Index="2" Names="UiConstants.AdminNavNames" Links="UiConstants.AdminNavLinks"/>
 </div>
 
-<FileManager FileSystemProvider="FileSystemProvider"/>
+<FileManager FileSystemProvider="FileSystemProvider" MaxUploadSize="4096"/>
 
 @code
 {
@@ -29,8 +27,7 @@
         FileSystemProvider = new SysFileSystemProvider(
             ApiClient,
             DownloadService,
-            LocalStorageService,
-            XmlHttpClient
+            LocalStorageService
         );
     }
 }

Moonlight.Client/wwwroot/index.html

@@ -32,7 +32,6 @@
     <script src="/js/fileManager.js"></script>
     <script src="/js/codeEditor.js"></script>
     <script src="/js/keyBinds.js"></script>
-    <script src="/js/xmlHttpRequest.js"></script>
     <script src="/ace/ace.js"></script>
     <script src="/_framework/blazor.webassembly.js"></script>
     <script>navigator.serviceWorker.register('service-worker.js');</script>

Moonlight.Client/wwwroot/js/downloadService.js

@@ -19,7 +19,7 @@ window.moonCoreDownloadService = {
                     const now = Date.now();
 
                     if (now - lastReportTime >= 500) { // Only log once per second
-                        await reportRef.invokeMethodAsync("ReceiveReport", id, receivedLength, false);
+                        await reportRef.invokeMethodAsync("ReceiveReport", id, receivedLength, -1, false);
                         lastReportTime = now;
                     }
                 }
@@ -31,7 +31,7 @@ window.moonCoreDownloadService = {
             this.downloadBlob(fileName, blob);
 
             if (reportRef)
-                await reportRef.invokeMethodAsync("ReceiveReport", id, receivedLength, true);
+                await reportRef.invokeMethodAsync("ReceiveReport", id, receivedLength, -1, true);
 
             resolve();
         });
@@ -56,13 +56,13 @@ window.moonCoreDownloadService = {
                     const now = Date.now();
 
                     if (now - lastReported >= 500) {
-                        await reportRef.invokeMethodAsync("ReceiveReport", id, ev.loaded, false);
+                        await reportRef.invokeMethodAsync("ReceiveReport", id, ev.loaded, ev.total, false);
                         lastReported = now;
                     }
                 };
 
                 loadRequest.onloadend = async ev => {
-                    await reportRef.invokeMethodAsync("ReceiveReport", id, ev.loaded, true);
+                    await reportRef.invokeMethodAsync("ReceiveReport", id, ev.loaded, ev.total, true);
                 }
             }
 

Moonlight.Client/wwwroot/js/xmlHttpRequest.js

@@ -1,80 +0,0 @@
-window.moonCoreXmlHttpRequest = {
-    storage: {},
-    initialize: function (trackingId, refObject) {
-        const req = new XMLHttpRequest();
-
-        req.addEventListener("timeout", async ev => {
-            await refObject.invokeMethodAsync("TriggerTimeoutEvent", {
-                "loaded": ev.loaded,
-                "total": ev.total
-            });
-        });
-
-        req.addEventListener("progress", async ev => {
-            await refObject.invokeMethodAsync("TriggerDownloadProgressEvent", {
-                "loaded": ev.loaded,
-                "total": ev.total
-            });
-        });
-
-        req.upload.addEventListener("progress", async ev => {
-            await refObject.invokeMethodAsync("TriggerUploadProgressEvent", {
-                "loaded": ev.loaded,
-                "total": ev.total
-            });
-        });
-
-        req.addEventListener("loadend", async ev => {
-            await refObject.invokeMethodAsync("TriggerLoadedEvent", ev);
-        });
-
-        req.addEventListener("readystatechange", async _ => {
-            await refObject.invokeMethodAsync("TriggerReadyStateChangeEvent", req.readyState);
-        });
-
-        this.storage[trackingId] = req;
-
-        return req;
-    },
-    setProperty: function (trackingId, property, value) {
-        this.storage[trackingId][property] = value;
-        console.log(this.storage[trackingId]);
-    },
-    getProperty: function (trackingId, property) {
-        return this.storage[trackingId][property];
-    },
-    sendStream: async function (trackingId, streamRef) {
-        const stream = await streamRef.stream();
-        const blob = await this.streamToBlob(stream);
-        this.storage[trackingId].send(blob);
-    },
-    sendFile: async function (trackingId, formName, fileName, streamRef) {
-        const stream = await streamRef.stream();
-        const blob = await this.streamToBlob(stream);
-
-        const formData = new FormData();
-        formData.append(formName, blob, fileName);
-
-        this.storage[trackingId].send(formData);
-    },
-    getResponseStream: function (trackingId) {
-        return this.storage[trackingId].response;
-    },
-    dispose: function (trackingId) {
-        this.storage[trackingId] = undefined;
-    },
-    streamToBlob: async function (stream) {
-        const reader = stream.getReader();
-
-        let chunks = [];
-
-        while (true) {
-            const {done, value} = await reader.read();
-            if (done) break;
-
-            chunks.push(value);
-        }
-
-        return new Blob(chunks);
-    }
-}