Moonlight-Panel/Servers
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implemented zip and tar compressing and decompressing. Implemented chunked file uploading

Browse Source
This commit is contained in:
ChiaraBm
2025-03-24 22:15:05 +01:00
parent 4046579c42
commit f56f94a03b
18 changed files with 573 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

256
MoonlightServers.Daemon/Helpers/ServerFileSystem.cs
View File

@@ -1,6 +1,12 @@
using System.Text;
using ICSharpCode.SharpZipLib.GZip;
using ICSharpCode.SharpZipLib.Tar;
using ICSharpCode.SharpZipLib.Zip;
using Mono.Unix.Native;
using MoonCore.Unix.Exceptions;
using MoonCore.Unix.SecureFs;
using MoonlightServers.DaemonShared.DaemonSide.Http.Responses.Servers;
using MoonlightServers.DaemonShared.Enums;
namespace MoonlightServers.Daemon.Helpers;
@@ -36,49 +42,78 @@ public class ServerFileSystem
{
var oldPath = Normalize(inputOldPath);
var newPath = Normalize(inputNewPath);
FileSystem.Rename(oldPath, newPath);
return Task.CompletedTask;
}
public Task Delete(string inputPath)
{
var path = Normalize(inputPath);
FileSystem.RemoveAll(path);
return Task.CompletedTask;
}
public Task Mkdir(string inputPath)
{
var path = Normalize(inputPath);
FileSystem.MkdirAll(path, FilePermissions.ACCESSPERMS);
return Task.CompletedTask;
}
public Task CreateChunk(string inputPath, long totalSize, long positionToSkip, Stream chunkStream)
{
var path = Normalize(inputPath);
var parentDirectory = Path.GetDirectoryName(path);
if (!string.IsNullOrEmpty(parentDirectory) && parentDirectory != "/")
FileSystem.MkdirAll(parentDirectory, FilePermissions.ACCESSPERMS);
FileSystem.OpenFileWrite(path, fileStream =>
{
if (fileStream.Length != totalSize)
fileStream.SetLength(totalSize);
fileStream.Position = positionToSkip;
chunkStream.CopyTo(fileStream);
fileStream.Flush();
}, OpenFlags.O_CREAT | OpenFlags.O_RDWR); // We use these custom flags to ensure we aren't overwriting the file
return Task.CompletedTask;
}
public Task Create(string inputPath, Stream dataStream)
{
var path = Normalize(inputPath);
var parentDirectory = Path.GetDirectoryName(path);
if(!string.IsNullOrEmpty(parentDirectory) && parentDirectory != "/")
if (!string.IsNullOrEmpty(parentDirectory) && parentDirectory != "/")
FileSystem.MkdirAll(parentDirectory, FilePermissions.ACCESSPERMS);
FileSystem.WriteFile(path, dataStream);
FileSystem.OpenFileWrite(path, stream =>
{
stream.Position = 0;
dataStream.CopyTo(stream);
stream.Flush();
});
return Task.CompletedTask;
}
public Task Read(string inputPath, Func<Stream, Task> onHandle)
{
var path = Normalize(inputPath);
FileSystem.OpenFile(path, stream =>
FileSystem.OpenFileRead(path, stream =>
{
// No try catch here because the safe fs abstraction already handles every error occuring in the handle
onHandle.Invoke(stream).Wait();
@@ -87,6 +122,201 @@ public class ServerFileSystem
return Task.CompletedTask;
}
#region Compression
public Task Compress(string[] itemsInput, string destinationInput, CompressType type)
{
var destination = Normalize(destinationInput);
var items = itemsInput.Select(Normalize);
if (type == CompressType.Zip)
{
FileSystem.OpenFileWrite(destination, stream =>
{
using var zipStream = new ZipOutputStream(stream);
foreach (var item in items)
AddItemToZip(item, zipStream);
zipStream.Flush();
stream.Flush();
zipStream.Close();
});
}
else if (type == CompressType.TarGz)
{
FileSystem.OpenFileWrite(destination, stream =>
{
using var gzStream = new GZipOutputStream(stream);
using var tarStream = new TarOutputStream(gzStream, Encoding.UTF8);
foreach (var item in items)
AddItemToTar(item, tarStream);
tarStream.Flush();
gzStream.Flush();
stream.Flush();
tarStream.Close();
gzStream.Close();
});
}
return Task.CompletedTask;
}
public Task Decompress(string pathInput, string destinationInput, CompressType type)
{
var path = Normalize(pathInput);
var destination = Normalize(destinationInput);
if (type == CompressType.Zip)
{
FileSystem.OpenFileRead(path, fileStream =>
{
var zipInputStream = new ZipInputStream(fileStream);
ExtractZip(zipInputStream, destination);
});
}
else if (type == CompressType.TarGz)
{
FileSystem.OpenFileRead(path, fileStream =>
{
var gzInputStream = new GZipInputStream(fileStream);
var zipInputStream = new TarInputStream(gzInputStream, Encoding.UTF8);
ExtractTar(zipInputStream, destination);
});
}
return Task.CompletedTask;
}
private void AddItemToZip(string path, ZipOutputStream outputStream)
{
var item = FileSystem.Stat(path);
if (item.IsDirectory)
{
var contents = FileSystem.ReadDir(path);
foreach (var content in contents)
{
AddItemToZip(
Path.Combine(path, content.Name),
outputStream
);
}
}
else
{
var entry = new ZipEntry(path)
{
Size = item.Size,
DateTime = item.LastChanged
};
outputStream.PutNextEntry(entry);
FileSystem.OpenFileRead(path, stream => { stream.CopyTo(outputStream); });
outputStream.CloseEntry();
}
}
private void AddItemToTar(string path, TarOutputStream outputStream)
{
var item = FileSystem.Stat(path);
if (item.IsDirectory)
{
var contents = FileSystem.ReadDir(path);
foreach (var content in contents)
{
AddItemToTar(
Path.Combine(path, content.Name),
outputStream
);
}
}
else
{
var entry = TarEntry.CreateTarEntry(path);
entry.Name = path;
entry.Size = item.Size;
entry.ModTime = item.LastChanged;
outputStream.PutNextEntry(entry);
FileSystem.OpenFileRead(path, stream =>
{
stream.CopyTo(outputStream);
});
outputStream.CloseEntry();
}
}
private void ExtractZip(ZipInputStream inputStream, string destination)
{
while (true)
{
var entry = inputStream.GetNextEntry();
if(entry == null || entry.IsDirectory)
break;
var fileDestination = Path.Combine(destination, entry.Name);
var parentDirectory = Path.GetDirectoryName(fileDestination);
if (!string.IsNullOrEmpty(parentDirectory) && parentDirectory != "/")
FileSystem.MkdirAll(parentDirectory, FilePermissions.ACCESSPERMS);
FileSystem.OpenFileWrite(fileDestination, stream =>
{
stream.Position = 0;
inputStream.CopyTo(stream);
stream.Flush();
}); // This will override the file if it exists
}
}
private void ExtractTar(TarInputStream inputStream, string destination)
{
while (true)
{
var entry = inputStream.GetNextEntry();
if(entry == null || entry.IsDirectory)
break;
var fileDestination = Path.Combine(destination, entry.Name);
var parentDirectory = Path.GetDirectoryName(fileDestination);
if (!string.IsNullOrEmpty(parentDirectory) && parentDirectory != "/")
FileSystem.MkdirAll(parentDirectory, FilePermissions.ACCESSPERMS);
FileSystem.OpenFileWrite(fileDestination, stream =>
{
stream.Position = 0;
inputStream.CopyTo(stream);
stream.Flush();
}); // This will override the file if it exists
}
}
#endregion
private string Normalize(string path)
{
return path

31
MoonlightServers.Daemon/Http/Controllers/Servers/ServerFileSystemController.cs
View File

@@ -2,6 +2,7 @@ using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using MoonCore.Exceptions;
using MoonlightServers.Daemon.Services;
using MoonlightServers.DaemonShared.DaemonSide.Http.Requests;
using MoonlightServers.DaemonShared.DaemonSide.Http.Responses.Servers;
namespace MoonlightServers.Daemon.Http.Controllers.Servers;
@@ -61,4 +62,34 @@ public class ServerFileSystemController : Controller
await server.FileSystem.Mkdir(path);
}
[HttpPost("{id:int}/files/compress")]
public async Task Compress([FromRoute] int id, [FromBody] ServerFilesCompressRequest request)
{
var server = ServerService.GetServer(id);
if (server == null)
throw new HttpApiException("No server with this id found", 404);
await server.FileSystem.Compress(
request.Items,
request.Destination,
request.Type
);
}
[HttpPost("{id:int}/files/decompress")]
public async Task Decompress([FromRoute] int id, [FromBody] ServerFilesDecompressRequest request)
{
var server = ServerService.GetServer(id);
if (server == null)
throw new HttpApiException("No server with this id found", 404);
await server.FileSystem.Decompress(
request.Path,
request.Destination,
request.Type
);
}
}

43
MoonlightServers.Daemon/Http/Controllers/Servers/UploadController.cs
View File

@@ -17,6 +17,8 @@ public class UploadController : Controller
private readonly AppConfiguration Configuration;
private readonly ServerService ServerService;
private readonly long ChunkSize = ByteConverter.FromMegaBytes(20).Bytes; // TODO config
public UploadController(
AccessTokenHelper accessTokenHelper,
ServerService serverService,
@@ -29,15 +31,24 @@ public class UploadController : Controller
}
[HttpPost]
public async Task Upload([FromQuery] string token)
public async Task Upload(
[FromQuery] string token,
[FromQuery] long totalSize, // TODO: Add limit in config
[FromQuery] int chunkId,
[FromQuery] string path
)
{
var file = Request.Form.Files.FirstOrDefault();
#region File validation
if (file == null)
throw new HttpApiException("No file provided", 400);
if (Request.Form.Files.Count != 1)
throw new HttpApiException("You need to provide exactly one file", 400);
var file = Request.Form.Files[0];
if(file.Length > ByteConverter.FromMegaBytes(Configuration.Files.UploadLimit).Bytes)
throw new HttpApiException("The provided file is bigger than the upload limit", 400);
if (file.Length > ChunkSize)
throw new HttpApiException("The provided data exceeds the chunk size limit", 400);
#endregion
#region Token validation
@@ -56,14 +67,30 @@ public class UploadController : Controller
#endregion
#region Chunk calculation and validation
var chunks = totalSize / ChunkSize;
chunks += totalSize % ChunkSize > 0 ? 1 : 0;
if (chunkId > chunks)
throw new HttpApiException("Invalid chunk id: Out of bounds", 400);
var positionToSkipTo = ChunkSize * chunkId;
#endregion
var server = ServerService.GetServer(serverId);
if (server == null)
throw new HttpApiException("No server with this id found", 404);
var dataStream = file.OpenReadStream();
var path = file.FileName;
await server.FileSystem.Create(path, dataStream);
await server.FileSystem.CreateChunk(
path,
totalSize,
positionToSkipTo,
dataStream
);
}
}

7
MoonlightServers.Daemon/MoonlightServers.Daemon.csproj
View File

@@ -9,9 +9,10 @@
<ItemGroup>
<PackageReference Include="Docker.DotNet" Version="3.125.15" />
<PackageReference Include="Microsoft.AspNetCore.SignalR" Version="1.1.0" />
<PackageReference Include="MoonCore" Version="1.8.4" />
<PackageReference Include="MoonCore.Extended" Version="1.3.0" />
<PackageReference Include="MoonCore.Unix" Version="1.0.3" />
<PackageReference Include="MoonCore" Version="1.8.5" />
<PackageReference Include="MoonCore.Extended" Version="1.3.2" />
<PackageReference Include="MoonCore.Unix" Version="1.0.6" />
<PackageReference Include="SharpZipLib" Version="1.4.2" />
<PackageReference Include="Stateless" Version="5.17.0" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.6.2"/>
</ItemGroup>