diff --git a/MoonlightServers.ApiServer/Http/Controllers/Admin/Servers/ServersController.cs b/MoonlightServers.ApiServer/Http/Controllers/Admin/Servers/ServersController.cs index b5f15bf..6da1bc4 100644 --- a/MoonlightServers.ApiServer/Http/Controllers/Admin/Servers/ServersController.cs +++ b/MoonlightServers.ApiServer/Http/Controllers/Admin/Servers/ServersController.cs @@ -193,6 +193,8 @@ public class ServersController : Controller [HttpPatch("{id:int}")] public async Task Update([FromRoute] int id, [FromBody] UpdateServerRequest request) { + //TODO: Handle shrinking virtual disk + var server = await CrudHelper.GetSingleModel(id); server = Mapper.Map(server, request); diff --git a/MoonlightServers.Daemon/Extensions/ServerConfigurationExtensions.cs b/MoonlightServers.Daemon/Extensions/ServerConfigurationExtensions.cs index 0cb0e50..9fffc2d 100644 --- a/MoonlightServers.Daemon/Extensions/ServerConfigurationExtensions.cs +++ b/MoonlightServers.Daemon/Extensions/ServerConfigurationExtensions.cs @@ -88,6 +88,14 @@ public static class ServerConfigurationExtensions var userId = Syscall.getuid(); // TODO: Extract to external service? + if (userId == 0) + userId = 998; + + parameters.User = $"{userId}:{userId}"; + + Console.WriteLine($"DUID: {userId}"); + + /* if (userId == 0) { // We are running as root, so we need to run the container as another user and chown the files when we make changes @@ -98,7 +106,7 @@ public static class ServerConfigurationExtensions // We are not running as root, so we start the container as the same user, // as we are not able to chown the container content to a different user parameters.User = $"{userId}:{userId}"; - } + }*/ #endregion diff --git a/MoonlightServers.Daemon/ServerSystem/Server.cs b/MoonlightServers.Daemon/ServerSystem/Server.cs index f778778..8921a73 100644 --- a/MoonlightServers.Daemon/ServerSystem/Server.cs +++ b/MoonlightServers.Daemon/ServerSystem/Server.cs @@ -20,8 +20,7 @@ public class Server : IAsyncDisposable private readonly IServiceScope ServiceScope; private readonly ILoggerFactory LoggerFactory; private readonly ILogger Logger; - - + public Server( ServerConfiguration configuration, IServiceScope serviceScope, diff --git a/MoonlightServers.Daemon/ServerSystem/SubSystems/ConsoleSubSystem.cs b/MoonlightServers.Daemon/ServerSystem/SubSystems/ConsoleSubSystem.cs index 866f149..be2b020 100644 --- a/MoonlightServers.Daemon/ServerSystem/SubSystems/ConsoleSubSystem.cs +++ b/MoonlightServers.Daemon/ServerSystem/SubSystems/ConsoleSubSystem.cs @@ -32,13 +32,19 @@ public class ConsoleSubSystem : ServerSubSystem { OnInput += async content => { - if(Stream == null) + if (Stream == null) return; - + var contentBuffer = Encoding.UTF8.GetBytes(content); - await Stream.WriteAsync(contentBuffer, 0, contentBuffer.Length, Server.TaskCancellation); + + await Stream.WriteAsync( + contentBuffer, + 0, + contentBuffer.Length, + Server.TaskCancellation + ); }; - + return Task.CompletedTask; } @@ -95,10 +101,10 @@ public class ConsoleSubSystem : ServerSubSystem Logger.LogWarning("An unhandled error occured while reading from container stream: {e}", e); } } - + // Reset stream so no further inputs will be piped to it Stream = null; - + Logger.LogDebug("Disconnected from container stream"); }); } @@ -124,7 +130,8 @@ public class ConsoleSubSystem : ServerSubSystem public async Task WriteMoonlight(string output) { - await WriteOutput($"\x1b[38;5;16;48;5;135m\x1b[39m\x1b[1m Moonlight \x1b[0m\x1b[38;5;250m\x1b[3m {output}\x1b[0m\n\r"); + await WriteOutput( + $"\x1b[0;38;2;255;255;255;48;2;124;28;230m Moonlight \x1b[0m\x1b[38;5;250m\x1b[3m {output}\x1b[0m\n\r"); } public async Task WriteInput(string input) diff --git a/MoonlightServers.Daemon/ServerSystem/SubSystems/InstallationSubSystem.cs b/MoonlightServers.Daemon/ServerSystem/SubSystems/InstallationSubSystem.cs index dc03392..2f522ed 100644 --- a/MoonlightServers.Daemon/ServerSystem/SubSystems/InstallationSubSystem.cs +++ b/MoonlightServers.Daemon/ServerSystem/SubSystems/InstallationSubSystem.cs @@ -114,13 +114,13 @@ public class InstallationSubSystem : ServerSubSystem var installData = await RemoteService.GetServerInstallation(Configuration.Id); - // 3. Ensure the storage location exists + // 3. Ensure the storage locations exists Logger.LogDebug("Ensuring storage"); var storageSubSystem = Server.GetRequiredSubSystem(); - if (!await storageSubSystem.IsRuntimeVolumeReady()) + if (!await storageSubSystem.RequestRuntimeVolume()) { Logger.LogDebug("Unable to continue provision because the server file system isn't ready"); await consoleSubSystem.WriteMoonlight("Server file system is not ready yet. Try again later"); @@ -129,9 +129,10 @@ public class InstallationSubSystem : ServerSubSystem return; } - var runtimePath = await storageSubSystem.GetRuntimeHostPath(); - - var installPath = await storageSubSystem.EnsureInstallVolume(); + var runtimePath = storageSubSystem.RuntimeVolumePath; + + await storageSubSystem.EnsureInstallVolume(); + var installPath = storageSubSystem.InstallVolumePath; // 4. Copy script to location diff --git a/MoonlightServers.Daemon/ServerSystem/SubSystems/ProvisionSubSystem.cs b/MoonlightServers.Daemon/ServerSystem/SubSystems/ProvisionSubSystem.cs index aeeb503..10f98f4 100644 --- a/MoonlightServers.Daemon/ServerSystem/SubSystems/ProvisionSubSystem.cs +++ b/MoonlightServers.Daemon/ServerSystem/SubSystems/ProvisionSubSystem.cs @@ -118,7 +118,7 @@ public class ProvisionSubSystem : ServerSubSystem var storageSubSystem = Server.GetRequiredSubSystem(); - if (!await storageSubSystem.IsRuntimeVolumeReady()) + if (!await storageSubSystem.RequestRuntimeVolume()) { Logger.LogDebug("Unable to continue provision because the server file system isn't ready"); await consoleSubSystem.WriteMoonlight("Server file system is not ready yet. Try again later"); @@ -127,7 +127,7 @@ public class ProvisionSubSystem : ServerSubSystem return; } - var volumePath = await storageSubSystem.GetRuntimeHostPath(); + var volumePath = storageSubSystem.RuntimeVolumePath; // 4. Ensure the docker image is downloaded diff --git a/MoonlightServers.Daemon/ServerSystem/SubSystems/StorageSubSystem.cs b/MoonlightServers.Daemon/ServerSystem/SubSystems/StorageSubSystem.cs index c8e7113..7f8b093 100644 --- a/MoonlightServers.Daemon/ServerSystem/SubSystems/StorageSubSystem.cs +++ b/MoonlightServers.Daemon/ServerSystem/SubSystems/StorageSubSystem.cs @@ -1,4 +1,7 @@ +using System.Diagnostics; +using Mono.Unix.Native; using MoonCore.Exceptions; +using MoonCore.Helpers; using MoonCore.Unix.SecureFs; using MoonlightServers.Daemon.Configuration; using MoonlightServers.Daemon.Helpers; @@ -8,9 +11,15 @@ namespace MoonlightServers.Daemon.ServerSystem.SubSystems; public class StorageSubSystem : ServerSubSystem { private readonly AppConfiguration AppConfiguration; - private SecureFileSystem SecureFileSystem; + private SecureFileSystem? SecureFileSystem; private ServerFileSystem ServerFileSystem; - private bool IsInitialized = false; + private ConsoleSubSystem ConsoleSubSystem; + + public string RuntimeVolumePath { get; private set; } + public string InstallVolumePath { get; private set; } + public string VirtualDiskPath { get; private set; } + public bool IsInitialized { get; private set; } + public bool IsVirtualDiskMounted { get; private set; } public StorageSubSystem( Server server, @@ -19,30 +28,65 @@ public class StorageSubSystem : ServerSubSystem ) : base(server, logger) { AppConfiguration = appConfiguration; + + // Runtime Volume + var runtimePath = Path.Combine(AppConfiguration.Storage.Volumes, Configuration.Id.ToString()); + + if (!runtimePath.StartsWith('/')) + runtimePath = Path.Combine(Directory.GetCurrentDirectory(), runtimePath); + + RuntimeVolumePath = runtimePath; + + // Install Volume + var installPath = Path.Combine(AppConfiguration.Storage.Install, Configuration.Id.ToString()); + + if (!installPath.StartsWith('/')) + installPath = Path.Combine(Directory.GetCurrentDirectory(), installPath); + + InstallVolumePath = installPath; + + // Virtual Disk + if (!Configuration.UseVirtualDisk) + return; + + var virtualDiskPath = Path.Combine(AppConfiguration.Storage.VirtualDisks, $"{Configuration.Id}.img"); + + if (!virtualDiskPath.StartsWith('/')) + virtualDiskPath = Path.Combine(Directory.GetCurrentDirectory(), virtualDiskPath); + + VirtualDiskPath = virtualDiskPath; } public override Task Initialize() { Logger.LogDebug("Lazy initializing server file system"); - + + ConsoleSubSystem = Server.GetRequiredSubSystem(); + Task.Run(async () => { try { await EnsureRuntimeVolume(); - var hostPath = await GetRuntimeHostPath(); - SecureFileSystem = new(hostPath); - ServerFileSystem = new(SecureFileSystem); - + // If we don't use a virtual disk the EnsureRuntimeVolume() method is + // all we need in order to serve access to the file system + if (!Configuration.UseVirtualDisk) + await CreateFileSystemAccessor(); + IsInitialized = true; } catch (Exception e) { + var consoleSubSystem = Server.GetRequiredSubSystem(); + + await consoleSubSystem.WriteMoonlight( + "Unable to initialize server file system. Please contact the administrator"); + Logger.LogError("An unhandled error occured while lazy initializing server file system: {e}", e); } }); - + return Task.CompletedTask; } @@ -54,112 +98,225 @@ public class StorageSubSystem : ServerSubSystem #region Runtime - public Task GetFileSystem() + public async Task GetFileSystem() { - if (!IsInitialized) + if (!await RequestRuntimeVolume(skipPermissions: true)) throw new HttpApiException("The file system is still initializing. Please try again later", 503); - return Task.FromResult(ServerFileSystem); + return ServerFileSystem; } - public Task IsRuntimeVolumeReady() + // This method allows other sub systems to request access to the runtime volume. + // The return value specifies if the request to the runtime volume is possible or not + public async Task RequestRuntimeVolume(bool skipPermissions = false) { - return Task.FromResult(IsInitialized); + if (!IsInitialized) + return false; + + if (!Configuration.UseVirtualDisk) + return true; // This is the default return for all servers without a virtual disk which has been initialized + + // If the disk isn't already mounted, we need to mount it now + if (!IsVirtualDiskMounted) + { + await MountVirtualDisk(); + + // And in order to serve the file system we need to create the accessor for it + await CreateFileSystemAccessor(); + } + + if(!skipPermissions) + await EnsureRuntimePermissions(); + + return IsVirtualDiskMounted; } private async Task EnsureRuntimeVolume() { - var path = await GetRuntimeHostPath(); - - if (!Directory.Exists(path)) - Directory.CreateDirectory(path); -/* - var consoleSubSystem = Server.GetRequiredSubSystem(); + if (!Directory.Exists(RuntimeVolumePath)) + Directory.CreateDirectory(RuntimeVolumePath); - await consoleSubSystem.WriteMoonlight("Creating virtual disk file. Please be patient"); - await Task.Delay(TimeSpan.FromSeconds(8)); - - await consoleSubSystem.WriteMoonlight("Formatting virtual disk. This can take a bit"); - await Task.Delay(TimeSpan.FromSeconds(8)); - - await consoleSubSystem.WriteMoonlight("Mounting virtual disk. Please be patient"); - await Task.Delay(TimeSpan.FromSeconds(3)); - - await consoleSubSystem.WriteMoonlight("Virtual disk ready");*/ - - // TODO: Implement virtual disk - } - - public Task GetRuntimeHostPath() - { - var path = Path.Combine( - AppConfiguration.Storage.Volumes, - Configuration.Id.ToString() - ); - - if (!path.StartsWith('/')) - path = Path.Combine(Directory.GetCurrentDirectory(), path); - - return Task.FromResult(path); + if (Configuration.UseVirtualDisk) + await EnsureVirtualDiskCreated(); } private async Task DeleteRuntimeVolume() { - var path = await GetRuntimeHostPath(); - - if(!Directory.Exists(path)) + if (!Directory.Exists(RuntimeVolumePath)) return; + + if (Configuration.UseVirtualDisk) + { + if (IsVirtualDiskMounted) + { + // Ensure the secure file system is no longer open + if(SecureFileSystem != null && !SecureFileSystem.IsDisposed) + SecureFileSystem.Dispose(); + + await UnmountVirtualDisk(); + } + + File.Delete(VirtualDiskPath); + } + else + { + if (SecureFileSystem == null) // If we are not already initialized, we are initializing now just the part we need + SecureFileSystem = new SecureFileSystem(RuntimeVolumePath); + + foreach (var entry in SecureFileSystem.ReadDir("/")) + { + if(entry.IsFile) + SecureFileSystem.Remove(entry.Name); + else + SecureFileSystem.RemoveAll(entry.Name); + } + + SecureFileSystem.Dispose(); + } + + Directory.Delete(RuntimeVolumePath, true); + } + + private Task EnsureRuntimePermissions() + { + ArgumentNullException.ThrowIfNull(SecureFileSystem); - Directory.Delete(path, true); + //TODO: Config + var uid = (int)Syscall.getuid(); + var gid = (int)Syscall.getgid(); + + if (uid == 0) + { + uid = 998; + gid = 998; + } + + foreach (var entry in SecureFileSystem.ReadDir("/")) + { + if (entry.IsFile) + SecureFileSystem.Chown(entry.Name, uid, gid); + else + SecureFileSystem.ChownAll(entry.Name, uid, gid); + } + + Syscall.chown(RuntimeVolumePath, uid, gid); + + return Task.CompletedTask; + } + + private Task CreateFileSystemAccessor() + { + SecureFileSystem = new(RuntimeVolumePath); + ServerFileSystem = new(SecureFileSystem); + + return Task.CompletedTask; } #endregion #region Installation - public async Task EnsureInstallVolume() + public Task EnsureInstallVolume() { - var path = await GetInstallHostPath(); - - if (!Directory.Exists(path)) - Directory.CreateDirectory(path); - - return path; + if (!Directory.Exists(InstallVolumePath)) + Directory.CreateDirectory(InstallVolumePath); + + return Task.CompletedTask; } - - public Task GetInstallHostPath() + + public Task DeleteInstallVolume() { - var path = Path.Combine( - AppConfiguration.Storage.Install, - Configuration.Id.ToString() + if (!Directory.Exists(InstallVolumePath)) + return Task.CompletedTask; + + Directory.Delete(InstallVolumePath, true); + return Task.CompletedTask; + } + + #endregion + + #region Virtual disks + + private async Task MountVirtualDisk() + { + // Check if we need to mount the virtual disk + if (await ExecuteCommand("findmnt", RuntimeVolumePath) != 0) + { + await ConsoleSubSystem.WriteMoonlight("Mounting virtual disk. Please be patient"); + await ExecuteCommand("mount", $"-t auto -o loop {VirtualDiskPath} {RuntimeVolumePath}", true); + } + + IsVirtualDiskMounted = true; + } + + private async Task UnmountVirtualDisk() + { + // Check if we need to unmount the virtual disk + if (await ExecuteCommand("findmnt", RuntimeVolumePath) != 0) + return; + + await ExecuteCommand("umount", $"{RuntimeVolumePath}"); + } + + private async Task EnsureVirtualDiskCreated() + { + // TODO: Handle resize + + if(File.Exists(VirtualDiskPath)) + return; + + // Create the image file and adjust the size + await ConsoleSubSystem.WriteMoonlight("Creating virtual disk file. Please be patient"); + + var fileStream = File.Open(VirtualDiskPath, FileMode.CreateNew, FileAccess.Write, FileShare.None); + + fileStream.SetLength( + ByteConverter.FromMegaBytes(Configuration.Disk).Bytes ); - if (!path.StartsWith('/')) - path = Path.Combine(Directory.GetCurrentDirectory(), path); - - return Task.FromResult(path); + await fileStream.FlushAsync(); + fileStream.Close(); + await fileStream.DisposeAsync(); + + // Now we want to format it + await ConsoleSubSystem.WriteMoonlight("Formatting virtual disk. This can take a bit"); + + await ExecuteCommand("mkfs", $"-t ext4 {VirtualDiskPath}", true); } - public async Task DeleteInstallVolume() + private async Task ExecuteCommand(string command, string arguments, bool handleExitCode = false) { - var path = await GetInstallHostPath(); - - if(!Directory.Exists(path)) - return; - - Directory.Delete(path, true); + var psi = new ProcessStartInfo() + { + FileName = command, + Arguments = arguments, + RedirectStandardError = true, + RedirectStandardOutput = true + }; + + var process = Process.Start(psi); + + if (process == null) + throw new AggregateException("The spawned process reference is null"); + + await process.WaitForExitAsync(); + + if (process.ExitCode == 0 || !handleExitCode) + return process.ExitCode; + + var output = await process.StandardOutput.ReadToEndAsync(); + output += await process.StandardError.ReadToEndAsync(); + + throw new Exception($"The command {command} failed: {output}"); } #endregion public override ValueTask DisposeAsync() { - if (IsInitialized) - { - if(!SecureFileSystem.IsDisposed) - SecureFileSystem.Dispose(); - } - + if (SecureFileSystem != null && !SecureFileSystem.IsDisposed) + SecureFileSystem.Dispose(); + return ValueTask.CompletedTask; } } \ No newline at end of file diff --git a/MoonlightServers.Daemon/Services/ServerService.cs b/MoonlightServers.Daemon/Services/ServerService.cs index 9905e82..212b3f7 100644 --- a/MoonlightServers.Daemon/Services/ServerService.cs +++ b/MoonlightServers.Daemon/Services/ServerService.cs @@ -17,7 +17,7 @@ namespace MoonlightServers.Daemon.Services; [Singleton] public class ServerService : IHostedLifecycleService { - private readonly Dictionary Servers = new(); + private readonly ConcurrentDictionary Servers = new(); private readonly RemoteService RemoteService; private readonly DockerClient DockerClient; @@ -58,82 +58,9 @@ public class ServerService : IHostedLifecycleService else await Initialize(serverId); } - - public async Task InitializeAll() - { - var initialPage = await RemoteService.GetServers(0, 1); - - const int pageSize = 25; - var pages = (initialPage.TotalItems == 0 ? 0 : (initialPage.TotalItems - 1) / pageSize) + - 1; // The +1 is to handle the pages starting at 0 - - // Create and fill a queue with pages to initialize - var batchesLeft = new ConcurrentQueue(); - - for (var i = 0; i < pages; i++) - batchesLeft.Enqueue(i); - - var tasksCount = pages > 5 ? 5 : pages; - var tasks = new List(); - - Logger.LogInformation( - "Starting initialization for {count} server(s) with {tasksCount} worker(s)", - initialPage.TotalItems, - tasksCount - ); - - for (var i = 0; i < tasksCount; i++) - { - var id = i + 0; - var task = Task.Run(() => BatchRunner(batchesLeft, id)); - - tasks.Add(task); - } - - await Task.WhenAll(tasks); - - Logger.LogInformation("Initialization completed"); - } - - private async Task BatchRunner(ConcurrentQueue queue, int id) - { - while (!queue.IsEmpty) - { - if (!queue.TryDequeue(out var page)) - continue; - - await InitializeBatch(page, 25); - - Logger.LogDebug("Worker {id}: Finished initialization of page {page}", id, page); - } - - Logger.LogDebug("Worker {id}: Finished", id); - } - - private async Task InitializeBatch(int page, int pageSize) - { - var servers = await RemoteService.GetServers(page, pageSize); - - var configurations = servers.Items - .Select(x => x.ToServerConfiguration()) - .ToArray(); - - foreach (var configuration in configurations) - { - try - { - await Sync(configuration.Id, configuration); - } - catch (Exception e) - { - Logger.LogError( - "An unhandled error occured while initializing server {id}: {e}", - configuration.Id, - e - ); - } - } - } + + public Server? Find(int serverId) + => Servers.GetValueOrDefault(serverId); public async Task Initialize(int serverId) { @@ -143,9 +70,6 @@ public class ServerService : IHostedLifecycleService await Initialize(configuration); } - public Server? Find(int serverId) - => Servers.GetValueOrDefault(serverId); - public async Task Initialize(ServerConfiguration configuration) { var serverScope = ServiceProvider.CreateScope(); @@ -235,16 +159,95 @@ public class ServerService : IHostedLifecycleService private async Task DeleteServer_Unhandled(Server server) { - await server.DisposeAsync(); await server.Delete(); + await server.DisposeAsync(); - lock (Servers) - Servers.Remove(server.Configuration.Id); + Servers.Remove(server.Configuration.Id, out _); } + #region Batch Initialization + + public async Task InitializeAll() + { + var initialPage = await RemoteService.GetServers(0, 1); + + const int pageSize = 25; + var pages = (initialPage.TotalItems == 0 ? 0 : (initialPage.TotalItems - 1) / pageSize) + + 1; // The +1 is to handle the pages starting at 0 + + // Create and fill a queue with pages to initialize + var batchesLeft = new ConcurrentQueue(); + + for (var i = 0; i < pages; i++) + batchesLeft.Enqueue(i); + + var tasksCount = pages > 5 ? 5 : pages; + var tasks = new List(); + + Logger.LogInformation( + "Starting initialization for {count} server(s) with {tasksCount} worker(s)", + initialPage.TotalItems, + tasksCount + ); + + for (var i = 0; i < tasksCount; i++) + { + var id = i + 0; + var task = Task.Run(() => BatchRunner(batchesLeft, id)); + + tasks.Add(task); + } + + await Task.WhenAll(tasks); + + Logger.LogInformation("Initialization completed"); + } + + private async Task BatchRunner(ConcurrentQueue queue, int id) + { + while (!queue.IsEmpty) + { + if (!queue.TryDequeue(out var page)) + continue; + + await InitializeBatch(page, 25); + + Logger.LogDebug("Worker {id}: Finished initialization of page {page}", id, page); + } + + Logger.LogDebug("Worker {id}: Finished", id); + } + + private async Task InitializeBatch(int page, int pageSize) + { + var servers = await RemoteService.GetServers(page, pageSize); + + var configurations = servers.Items + .Select(x => x.ToServerConfiguration()) + .ToArray(); + + foreach (var configuration in configurations) + { + try + { + await Sync(configuration.Id, configuration); + } + catch (Exception e) + { + Logger.LogError( + "An unhandled error occured while initializing server {id}: {e}", + configuration.Id, + e + ); + } + } + } + + #endregion + #region Docker Monitoring - private async Task MonitorContainers() + private Task StartContainerMonitoring() { Task.Run(async () => { @@ -296,6 +299,8 @@ public class ServerService : IHostedLifecycleService } } }); + + return Task.CompletedTask; } #endregion @@ -310,7 +315,7 @@ public class ServerService : IHostedLifecycleService public async Task StartedAsync(CancellationToken cancellationToken) { - await MonitorContainers(); + await StartContainerMonitoring(); await InitializeAll(); }