Implemented zip and tar compressing and decompressing. Implemented chunked file uploading

This commit is contained in:
2025-03-24 22:15:05 +01:00
parent 4046579c42
commit f56f94a03b
18 changed files with 573 additions and 55 deletions

View File

@@ -6,6 +6,8 @@ using MoonCore.Extended.Abstractions;
using Moonlight.ApiServer.Database.Entities;
using MoonlightServers.ApiServer.Database.Entities;
using MoonlightServers.ApiServer.Services;
using MoonlightServers.DaemonShared.Enums;
using MoonlightServers.Shared.Http.Requests.Client.Servers.Files;
using MoonlightServers.Shared.Http.Responses.Client.Servers.Files;
namespace MoonlightServers.ApiServer.Http.Controllers.Client;
@@ -132,6 +134,28 @@ public class ServerFileSystemController : Controller
};
}
[HttpPost("{serverId:int}/files/compress")]
public async Task Compress([FromRoute] int serverId, [FromBody] ServerFilesCompressRequest request)
{
var server = await GetServerById(serverId);
if (!Enum.TryParse(request.Type, true, out CompressType type))
throw new HttpApiException("Invalid compress type provided", 400);
await ServerFileSystemService.Compress(server, type, request.Items, request.Destination);
}
[HttpPost("{serverId:int}/files/decompress")]
public async Task Decompress([FromRoute] int serverId, [FromBody] ServerFilesDecompressRequest request)
{
var server = await GetServerById(serverId);
if (!Enum.TryParse(request.Type, true, out CompressType type))
throw new HttpApiException("Invalid compress type provided", 400);
await ServerFileSystemService.Decompress(server, type, request.Path, request.Destination);
}
private async Task<Server> GetServerById(int serverId)
{
var server = await ServerRepository

View File

@@ -4,7 +4,9 @@ using MoonCore.Attributes;
using MoonCore.Extended.Abstractions;
using MoonCore.Helpers;
using MoonlightServers.ApiServer.Database.Entities;
using MoonlightServers.DaemonShared.DaemonSide.Http.Requests;
using MoonlightServers.DaemonShared.DaemonSide.Http.Responses.Servers;
using MoonlightServers.DaemonShared.Enums;
namespace MoonlightServers.ApiServer.Services;
@@ -59,6 +61,36 @@ public class ServerFileSystemService
);
}
public async Task Compress(Server server, CompressType type, string[] items, string destination)
{
using var apiClient = await GetApiClient(server);
await apiClient.Post(
$"api/servers/{server.Id}/files/compress",
new ServerFilesCompressRequest()
{
Type = type,
Items = items,
Destination = destination
}
);
}
public async Task Decompress(Server server, CompressType type, string path, string destination)
{
using var apiClient = await GetApiClient(server);
await apiClient.Post(
$"api/servers/{server.Id}/files/decompress",
new ServerFilesDecompressRequest()
{
Type = type,
Path = path,
Destination = destination
}
);
}
#region Helpers
private async Task<HttpApiClient> GetApiClient(Server server)

View File

@@ -1,6 +1,12 @@
using System.Text;
using ICSharpCode.SharpZipLib.GZip;
using ICSharpCode.SharpZipLib.Tar;
using ICSharpCode.SharpZipLib.Zip;
using Mono.Unix.Native;
using MoonCore.Unix.Exceptions;
using MoonCore.Unix.SecureFs;
using MoonlightServers.DaemonShared.DaemonSide.Http.Responses.Servers;
using MoonlightServers.DaemonShared.Enums;
namespace MoonlightServers.Daemon.Helpers;
@@ -60,6 +66,29 @@ public class ServerFileSystem
return Task.CompletedTask;
}
public Task CreateChunk(string inputPath, long totalSize, long positionToSkip, Stream chunkStream)
{
var path = Normalize(inputPath);
var parentDirectory = Path.GetDirectoryName(path);
if (!string.IsNullOrEmpty(parentDirectory) && parentDirectory != "/")
FileSystem.MkdirAll(parentDirectory, FilePermissions.ACCESSPERMS);
FileSystem.OpenFileWrite(path, fileStream =>
{
if (fileStream.Length != totalSize)
fileStream.SetLength(totalSize);
fileStream.Position = positionToSkip;
chunkStream.CopyTo(fileStream);
fileStream.Flush();
}, OpenFlags.O_CREAT | OpenFlags.O_RDWR); // We use these custom flags to ensure we aren't overwriting the file
return Task.CompletedTask;
}
public Task Create(string inputPath, Stream dataStream)
{
var path = Normalize(inputPath);
@@ -69,7 +98,13 @@ public class ServerFileSystem
if (!string.IsNullOrEmpty(parentDirectory) && parentDirectory != "/")
FileSystem.MkdirAll(parentDirectory, FilePermissions.ACCESSPERMS);
FileSystem.WriteFile(path, dataStream);
FileSystem.OpenFileWrite(path, stream =>
{
stream.Position = 0;
dataStream.CopyTo(stream);
stream.Flush();
});
return Task.CompletedTask;
}
@@ -78,7 +113,7 @@ public class ServerFileSystem
{
var path = Normalize(inputPath);
FileSystem.OpenFile(path, stream =>
FileSystem.OpenFileRead(path, stream =>
{
// No try catch here because the safe fs abstraction already handles every error occuring in the handle
onHandle.Invoke(stream).Wait();
@@ -87,6 +122,201 @@ public class ServerFileSystem
return Task.CompletedTask;
}
#region Compression
public Task Compress(string[] itemsInput, string destinationInput, CompressType type)
{
var destination = Normalize(destinationInput);
var items = itemsInput.Select(Normalize);
if (type == CompressType.Zip)
{
FileSystem.OpenFileWrite(destination, stream =>
{
using var zipStream = new ZipOutputStream(stream);
foreach (var item in items)
AddItemToZip(item, zipStream);
zipStream.Flush();
stream.Flush();
zipStream.Close();
});
}
else if (type == CompressType.TarGz)
{
FileSystem.OpenFileWrite(destination, stream =>
{
using var gzStream = new GZipOutputStream(stream);
using var tarStream = new TarOutputStream(gzStream, Encoding.UTF8);
foreach (var item in items)
AddItemToTar(item, tarStream);
tarStream.Flush();
gzStream.Flush();
stream.Flush();
tarStream.Close();
gzStream.Close();
});
}
return Task.CompletedTask;
}
public Task Decompress(string pathInput, string destinationInput, CompressType type)
{
var path = Normalize(pathInput);
var destination = Normalize(destinationInput);
if (type == CompressType.Zip)
{
FileSystem.OpenFileRead(path, fileStream =>
{
var zipInputStream = new ZipInputStream(fileStream);
ExtractZip(zipInputStream, destination);
});
}
else if (type == CompressType.TarGz)
{
FileSystem.OpenFileRead(path, fileStream =>
{
var gzInputStream = new GZipInputStream(fileStream);
var zipInputStream = new TarInputStream(gzInputStream, Encoding.UTF8);
ExtractTar(zipInputStream, destination);
});
}
return Task.CompletedTask;
}
private void AddItemToZip(string path, ZipOutputStream outputStream)
{
var item = FileSystem.Stat(path);
if (item.IsDirectory)
{
var contents = FileSystem.ReadDir(path);
foreach (var content in contents)
{
AddItemToZip(
Path.Combine(path, content.Name),
outputStream
);
}
}
else
{
var entry = new ZipEntry(path)
{
Size = item.Size,
DateTime = item.LastChanged
};
outputStream.PutNextEntry(entry);
FileSystem.OpenFileRead(path, stream => { stream.CopyTo(outputStream); });
outputStream.CloseEntry();
}
}
private void AddItemToTar(string path, TarOutputStream outputStream)
{
var item = FileSystem.Stat(path);
if (item.IsDirectory)
{
var contents = FileSystem.ReadDir(path);
foreach (var content in contents)
{
AddItemToTar(
Path.Combine(path, content.Name),
outputStream
);
}
}
else
{
var entry = TarEntry.CreateTarEntry(path);
entry.Name = path;
entry.Size = item.Size;
entry.ModTime = item.LastChanged;
outputStream.PutNextEntry(entry);
FileSystem.OpenFileRead(path, stream =>
{
stream.CopyTo(outputStream);
});
outputStream.CloseEntry();
}
}
private void ExtractZip(ZipInputStream inputStream, string destination)
{
while (true)
{
var entry = inputStream.GetNextEntry();
if(entry == null || entry.IsDirectory)
break;
var fileDestination = Path.Combine(destination, entry.Name);
var parentDirectory = Path.GetDirectoryName(fileDestination);
if (!string.IsNullOrEmpty(parentDirectory) && parentDirectory != "/")
FileSystem.MkdirAll(parentDirectory, FilePermissions.ACCESSPERMS);
FileSystem.OpenFileWrite(fileDestination, stream =>
{
stream.Position = 0;
inputStream.CopyTo(stream);
stream.Flush();
}); // This will override the file if it exists
}
}
private void ExtractTar(TarInputStream inputStream, string destination)
{
while (true)
{
var entry = inputStream.GetNextEntry();
if(entry == null || entry.IsDirectory)
break;
var fileDestination = Path.Combine(destination, entry.Name);
var parentDirectory = Path.GetDirectoryName(fileDestination);
if (!string.IsNullOrEmpty(parentDirectory) && parentDirectory != "/")
FileSystem.MkdirAll(parentDirectory, FilePermissions.ACCESSPERMS);
FileSystem.OpenFileWrite(fileDestination, stream =>
{
stream.Position = 0;
inputStream.CopyTo(stream);
stream.Flush();
}); // This will override the file if it exists
}
}
#endregion
private string Normalize(string path)
{
return path

View File

@@ -2,6 +2,7 @@ using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using MoonCore.Exceptions;
using MoonlightServers.Daemon.Services;
using MoonlightServers.DaemonShared.DaemonSide.Http.Requests;
using MoonlightServers.DaemonShared.DaemonSide.Http.Responses.Servers;
namespace MoonlightServers.Daemon.Http.Controllers.Servers;
@@ -61,4 +62,34 @@ public class ServerFileSystemController : Controller
await server.FileSystem.Mkdir(path);
}
[HttpPost("{id:int}/files/compress")]
public async Task Compress([FromRoute] int id, [FromBody] ServerFilesCompressRequest request)
{
var server = ServerService.GetServer(id);
if (server == null)
throw new HttpApiException("No server with this id found", 404);
await server.FileSystem.Compress(
request.Items,
request.Destination,
request.Type
);
}
[HttpPost("{id:int}/files/decompress")]
public async Task Decompress([FromRoute] int id, [FromBody] ServerFilesDecompressRequest request)
{
var server = ServerService.GetServer(id);
if (server == null)
throw new HttpApiException("No server with this id found", 404);
await server.FileSystem.Decompress(
request.Path,
request.Destination,
request.Type
);
}
}

View File

@@ -17,6 +17,8 @@ public class UploadController : Controller
private readonly AppConfiguration Configuration;
private readonly ServerService ServerService;
private readonly long ChunkSize = ByteConverter.FromMegaBytes(20).Bytes; // TODO config
public UploadController(
AccessTokenHelper accessTokenHelper,
ServerService serverService,
@@ -29,15 +31,24 @@ public class UploadController : Controller
}
[HttpPost]
public async Task Upload([FromQuery] string token)
public async Task Upload(
[FromQuery] string token,
[FromQuery] long totalSize, // TODO: Add limit in config
[FromQuery] int chunkId,
[FromQuery] string path
)
{
var file = Request.Form.Files.FirstOrDefault();
#region File validation
if (file == null)
throw new HttpApiException("No file provided", 400);
if (Request.Form.Files.Count != 1)
throw new HttpApiException("You need to provide exactly one file", 400);
if(file.Length > ByteConverter.FromMegaBytes(Configuration.Files.UploadLimit).Bytes)
throw new HttpApiException("The provided file is bigger than the upload limit", 400);
var file = Request.Form.Files[0];
if (file.Length > ChunkSize)
throw new HttpApiException("The provided data exceeds the chunk size limit", 400);
#endregion
#region Token validation
@@ -56,14 +67,30 @@ public class UploadController : Controller
#endregion
#region Chunk calculation and validation
var chunks = totalSize / ChunkSize;
chunks += totalSize % ChunkSize > 0 ? 1 : 0;
if (chunkId > chunks)
throw new HttpApiException("Invalid chunk id: Out of bounds", 400);
var positionToSkipTo = ChunkSize * chunkId;
#endregion
var server = ServerService.GetServer(serverId);
if (server == null)
throw new HttpApiException("No server with this id found", 404);
var dataStream = file.OpenReadStream();
var path = file.FileName;
await server.FileSystem.Create(path, dataStream);
await server.FileSystem.CreateChunk(
path,
totalSize,
positionToSkipTo,
dataStream
);
}
}

View File

@@ -9,9 +9,10 @@
<ItemGroup>
<PackageReference Include="Docker.DotNet" Version="3.125.15" />
<PackageReference Include="Microsoft.AspNetCore.SignalR" Version="1.1.0" />
<PackageReference Include="MoonCore" Version="1.8.4" />
<PackageReference Include="MoonCore.Extended" Version="1.3.0" />
<PackageReference Include="MoonCore.Unix" Version="1.0.3" />
<PackageReference Include="MoonCore" Version="1.8.5" />
<PackageReference Include="MoonCore.Extended" Version="1.3.2" />
<PackageReference Include="MoonCore.Unix" Version="1.0.6" />
<PackageReference Include="SharpZipLib" Version="1.4.2" />
<PackageReference Include="Stateless" Version="5.17.0" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.6.2"/>
</ItemGroup>

View File

@@ -0,0 +1,10 @@
using MoonlightServers.DaemonShared.Enums;
namespace MoonlightServers.DaemonShared.DaemonSide.Http.Requests;
public class ServerFilesCompressRequest
{
public string Destination { get; set; }
public string[] Items { get; set; }
public CompressType Type { get; set; }
}

View File

@@ -0,0 +1,10 @@
using MoonlightServers.DaemonShared.Enums;
namespace MoonlightServers.DaemonShared.DaemonSide.Http.Requests;
public class ServerFilesDecompressRequest
{
public string Destination { get; set; }
public string Path { get; set; }
public CompressType Type { get; set; }
}

View File

@@ -0,0 +1,7 @@
namespace MoonlightServers.DaemonShared.Enums;
public enum CompressType
{
Zip = 0,
TarGz = 1
}

View File

@@ -8,6 +8,7 @@
<ItemGroup>
<Folder Include="DaemonSide\" />
<Folder Include="DaemonSide\Http\Responses\Servers\Files\" />
</ItemGroup>
</Project>

View File

@@ -1,20 +1,41 @@
using MoonCore.Blazor.Tailwind.Fm;
using MoonCore.Blazor.Tailwind.Fm.Models;
using MoonCore.Blazor.Tailwind.Services;
using MoonCore.Helpers;
using MoonlightServers.Frontend.Services;
namespace MoonlightServers.Frontend.Helpers;
public class ServerFileSystemProvider : IFileSystemProvider
public class ServerFileSystemProvider : IFileSystemProvider, ICompressFileSystemProvider
{
private readonly int ServerId;
private readonly DownloadService DownloadService;
private readonly ServerFileSystemService FileSystemService;
public CompressType[] CompressTypes { get; } =
[
new()
{
Extension = "zip",
DisplayName = "ZIP Archive"
},
new()
{
Extension = "tar.gz",
DisplayName = "GZ Compressed Tar Archive"
}
];
private readonly int ServerId;
public ServerFileSystemProvider(
int serverId,
ServerFileSystemService fileSystemService
ServerFileSystemService fileSystemService,
DownloadService downloadService
)
{
ServerId = serverId;
FileSystemService = fileSystemService;
DownloadService = downloadService;
}
public async Task<FileSystemEntry[]> List(string path)
@@ -35,7 +56,7 @@ public class ServerFileSystemProvider : IFileSystemProvider
public async Task Create(string path, Stream stream)
{
await FileSystemService.Upload(ServerId, path, stream);
await Upload(_ => Task.CompletedTask, path, stream);
}
public async Task Move(string oldPath, string newPath)
@@ -55,6 +76,72 @@ public class ServerFileSystemProvider : IFileSystemProvider
public async Task<Stream> Read(string path)
{
return await FileSystemService.Download(ServerId, path);
var downloadSession = await FileSystemService.Download(ServerId, path);
using var httpClient = new HttpClient();
return await httpClient.GetStreamAsync(downloadSession.DownloadUrl);
}
public async Task Download(Func<int, Task> updateProgress, string path, string fileName)
{
var downloadSession = await FileSystemService.Download(ServerId, path);
await DownloadService.DownloadUrl(fileName, downloadSession.DownloadUrl,
async (loaded, total) =>
{
var percent = total == 0 ? 0 : (int)Math.Round((float)loaded / total * 100);
await updateProgress.Invoke(percent);
}
);
}
public async Task Upload(Func<int, Task> updateProgress, string path, Stream stream)
{
using var httpClient = new HttpClient();
var uploadSession = await FileSystemService.Upload(ServerId);
var size = stream.Length;
var chunkSize = ByteConverter.FromMegaBytes(20).Bytes;
var chunks = size / chunkSize;
chunks += size % chunkSize > 0 ? 1 : 0;
for (var chunkId = 0; chunkId < chunks; chunkId++)
{
var percent = (int)Math.Round((chunkId + 1f) / chunks * 100);
await updateProgress.Invoke(percent);
var buffer = new byte[chunkSize];
var bytesRead = await stream.ReadAsync(buffer);
var uploadForm = new MultipartFormDataContent();
uploadForm.Add(new ByteArrayContent(buffer, 0, bytesRead), "file", "file");
await httpClient.PostAsync(
$"{uploadSession.UploadUrl}&totalSize={size}&chunkId={chunkId}&path={path}",
uploadForm
);
}
}
public async Task Compress(CompressType type, string path, string[] itemsToCompress)
{
await FileSystemService.Compress(
ServerId,
type.Extension.Replace(".", ""),
itemsToCompress,
path
);
}
public async Task Decompress(CompressType type, string path, string destination)
{
await FileSystemService.Decompress(
ServerId,
type.Extension.Replace(".", ""),
path,
destination
);
}
}

View File

@@ -1,5 +1,7 @@
using System.Net;
using MoonCore.Attributes;
using MoonCore.Helpers;
using MoonlightServers.Shared.Http.Requests.Client.Servers.Files;
using MoonlightServers.Shared.Http.Responses.Client.Servers.Files;
namespace MoonlightServers.Frontend.Services;
@@ -42,28 +44,43 @@ public class ServerFileSystemService
);
}
public async Task Upload(int serverId, string path, Stream dataStream)
public async Task<ServerFilesUploadResponse> Upload(int serverId)
{
var uploadSession = await ApiClient.GetJson<ServerFilesUploadResponse>(
return await ApiClient.GetJson<ServerFilesUploadResponse>(
$"api/client/servers/{serverId}/files/upload"
);
using var httpClient = new HttpClient();
var content = new MultipartFormDataContent();
content.Add(new StreamContent(dataStream), "file", path);
await httpClient.PostAsync(uploadSession.UploadUrl, content);
}
public async Task<Stream> Download(int serverId, string path)
public async Task<ServerFilesDownloadResponse> Download(int serverId, string path)
{
var downloadSession = await ApiClient.GetJson<ServerFilesDownloadResponse>(
return await ApiClient.GetJson<ServerFilesDownloadResponse>(
$"api/client/servers/{serverId}/files/download?path={path}"
);
}
using var httpClient = new HttpClient();
public async Task Compress(int serverId, string type, string[] items, string destination)
{
await ApiClient.Post(
$"api/client/servers/{serverId}/files/compress",
new ServerFilesCompressRequest()
{
Type = type,
Items = items,
Destination = destination
}
);
}
return await httpClient.GetStreamAsync(downloadSession.DownloadUrl);
public async Task Decompress(int serverId, string type, string path, string destination)
{
await ApiClient.Post(
$"api/client/servers/{serverId}/files/decompress",
new ServerFilesDecompressRequest()
{
Type = type,
Path = path,
Destination = destination
}
);
}
}

View File

@@ -1,10 +1,12 @@
@using MoonlightServers.Frontend.Services
@using MoonCore.Blazor.Tailwind.Fm
@using MoonCore.Blazor.Tailwind.Services
@using MoonlightServers.Frontend.Helpers
@inherits BaseServerTab
@inject ServerFileSystemService FileSystemService
@inject DownloadService DownloadService
<FileManager FileSystemProvider="Provider"/>
@@ -14,6 +16,10 @@
protected override void OnInitialized()
{
Provider = new ServerFileSystemProvider(Server.Id, FileSystemService);
Provider = new ServerFileSystemProvider(
Server.Id,
FileSystemService,
DownloadService
);
}
}

View File

@@ -26,8 +26,10 @@
</PageHeader>
</div>
<DataTable @ref="Table" TItem="ServerDetailResponse" PageSize="15" LoadItemsPaginatedAsync="LoadData">
<DataTable @ref="Table" TItem="ServerDetailResponse">
<Configuration>
<Pagination TItem="ServerDetailResponse" ItemSource="LoadData" />
<DataTableColumn TItem="ServerDetailResponse" Field="@(x => x.Id)" Name="Id"/>
<DataTableColumn TItem="ServerDetailResponse" Field="@(x => x.Name)" Name="Name"/>
<DataTableColumn TItem="ServerDetailResponse" Field="@(x => x.NodeId)" Name="Node">

View File

@@ -27,8 +27,10 @@
</PageHeader>
</div>
<DataTable TItem="NodeDetailResponse" PageSize="15" LoadItemsPaginatedAsync="LoadData">
<DataTable TItem="NodeDetailResponse">
<Configuration>
<Pagination TItem="NodeDetailResponse" ItemSource="LoadData" />
<DataTableColumn TItem="NodeDetailResponse" Field="@(x => x.Id)" Name="Id"/>
<DataTableColumn TItem="NodeDetailResponse" Field="@(x => x.Name)" Name="Name">
<ColumnTemplate>

View File

@@ -33,8 +33,10 @@
</PageHeader>
</div>
<DataTable @ref="Table" TItem="StarDetailResponse" LoadItemsPaginatedAsync="LoadData">
<DataTable @ref="Table" TItem="StarDetailResponse">
<Configuration>
<Pagination TItem="StarDetailResponse" ItemSource="LoadData" />
<DataTableColumn TItem="StarDetailResponse" Field="@(x => x.Id)" Name="Id"/>
<DataTableColumn TItem="StarDetailResponse" Field="@(x => x.Name)" Name="Name">
<ColumnTemplate>

View File

@@ -0,0 +1,14 @@
using System.ComponentModel.DataAnnotations;
namespace MoonlightServers.Shared.Http.Requests.Client.Servers.Files;
public class ServerFilesCompressRequest
{
[Required(ErrorMessage = "You need to specify a type")]
public string Type { get; set; }
public string[] Items { get; set; } = [];
[Required(ErrorMessage = "You need to specify a destination")]
public string Destination { get; set; }
}

View File

@@ -0,0 +1,15 @@
using System.ComponentModel.DataAnnotations;
namespace MoonlightServers.Shared.Http.Requests.Client.Servers.Files;
public class ServerFilesDecompressRequest
{
[Required(ErrorMessage = "You need to specify a type")]
public string Type { get; set; }
[Required(ErrorMessage = "You need to specify a path")]
public string Path { get; set; }
[Required(ErrorMessage = "You need to specify a destination")]
public string Destination { get; set; }
}