Implemented installation handling. Added crash handling. Refactored tasks reset/cancel functions

MoonlightServers.Daemon/Abstractions/Server.Console.cs

@@ -72,7 +72,7 @@ public partial class Server
 
     private async Task LogToConsole(string message)
     {
-        await Console.WriteToOutput($"\x1b[38;5;16;48;5;135m\x1b[39m\x1b[1m Moonlight \x1b[0m {message}\n\r");
+        await Console.WriteToOutput($"\x1b[38;5;16;48;5;135m\x1b[39m\x1b[1m Moonlight \x1b[0m\x1b[38;5;250m\x1b[3m {message}\x1b[0m\n\r");
     }
 
     public Task<string[]> GetConsoleMessages()

MoonlightServers.Daemon/Abstractions/Server.Crash.cs

@@ -0,0 +1,34 @@
+using Docker.DotNet;
+using Docker.DotNet.Models;
+
+namespace MoonlightServers.Daemon.Abstractions;
+
+public partial class Server
+{
+    public async Task InternalCrash()
+    {
+        var dockerClient = ServiceProvider.GetRequiredService<DockerClient>();
+
+        ContainerInspectResponse? container;
+        
+        try
+        {
+            container = await dockerClient.Containers.InspectContainerAsync(RuntimeContainerId);
+        }
+        catch (DockerContainerNotFoundException)
+        {
+            container = null;
+        }
+        
+        if(container == null)
+            return;
+
+        var exitCode = container.State.ExitCode;
+        
+        // TODO: Report to panel
+        
+        await LogToConsole($"Server crashed. Exit code: {exitCode}");
+        
+        await Destroy();
+    }
+}

MoonlightServers.Daemon/Abstractions/Server.Destroy.cs

@@ -35,13 +35,19 @@ public partial class Server
         }
         catch (DockerContainerNotFoundException){}
         
-        // Canceling server tasks & listeners
+        // Canceling server tasks & listeners and start new ones
-        await CancelTasks();
+        await ResetTasks();
-        
-        // and recreating cancellation token
-        Cancellation = new();
     }
 
+    public async Task ResetTasks()
+    {
+        // Note: This will keep the docker container running, it will just cancel the server cancellation token
+        // and recreate the token
+        await CancelTasks();
+        
+        Cancellation = new();
+    }
+    
     public async Task CancelTasks()
     {
         // Note: This will keep the docker container running, it will just cancel the server cancellation token

MoonlightServers.Daemon/Abstractions/Server.Initialize.cs

@@ -1,6 +1,7 @@
 using System.Text.RegularExpressions;
 using Docker.DotNet.Models;
 using MoonlightServers.Daemon.Enums;
+using MoonlightServers.Daemon.Extensions;
 using Stateless;
 
 namespace MoonlightServers.Daemon.Abstractions;
@@ -38,23 +39,27 @@ public partial class Server
             .Permit(ServerTrigger.Reinstall, ServerState.Installing);
 
         StateMachine.Configure(ServerState.Starting)
-            .Permit(ServerTrigger.NotifyContainerDied, ServerState.Offline)
+            .Permit(ServerTrigger.NotifyRuntimeContainerDied, ServerState.Offline)
             .Permit(ServerTrigger.NotifyOnline, ServerState.Online)
             .Permit(ServerTrigger.Stop, ServerState.Stopping)
-            .OnEntryAsync(InternalStart);
+            .OnEntryAsync(InternalStart)
+            .OnExitFromAsync(ServerTrigger.NotifyRuntimeContainerDied, InternalCrash);
 
         StateMachine.Configure(ServerState.Online)
-            .Permit(ServerTrigger.NotifyContainerDied, ServerState.Offline)
+            .Permit(ServerTrigger.NotifyRuntimeContainerDied, ServerState.Offline)
-            .Permit(ServerTrigger.Stop, ServerState.Stopping);
+            .Permit(ServerTrigger.Stop, ServerState.Stopping)
+            .OnExitFromAsync(ServerTrigger.NotifyRuntimeContainerDied, InternalCrash);
 
         StateMachine.Configure(ServerState.Stopping)
-            .Permit(ServerTrigger.NotifyContainerDied, ServerState.Offline)
+            .Permit(ServerTrigger.NotifyRuntimeContainerDied, ServerState.Offline)
             .Permit(ServerTrigger.Kill, ServerState.Offline)
-            .OnEntryAsync(InternalStop);
+            .OnEntryAsync(InternalStop)
+            .OnExitFromAsync(ServerTrigger.NotifyRuntimeContainerDied, InternalFinishStop);
 
         StateMachine.Configure(ServerState.Installing)
-            .Permit(ServerTrigger.NotifyContainerDied, ServerState.Offline)
+            .Permit(ServerTrigger.NotifyInstallationContainerDied, ServerState.Offline)
-            .OnEntryAsync(InternalInstall);
+            .OnEntryAsync(InternalInstall)
+            .OnExitAsync(InternalFinishInstall);
 
         return Task.CompletedTask;
     }

MoonlightServers.Daemon/Abstractions/Server.Installation.cs

@@ -1,4 +1,5 @@
 using Docker.DotNet;
+using Docker.DotNet.Models;
 using MoonCore.Helpers;
 using MoonlightServers.Daemon.Enums;
 using MoonlightServers.Daemon.Extensions;
@@ -36,7 +37,8 @@ public partial class Server
         var runtimeHostPath = await EnsureRuntimeVolume();
         
         // Write installation script to path
-        await File.WriteAllTextAsync(PathBuilder.File(installationHostPath, "install.sh"), installData.Script.Replace("\n\r", "\n") + "\n\n");
+        var content = installData.Script.Replace("\r\n", "\n");
+        await File.WriteAllTextAsync(PathBuilder.File(installationHostPath, "install.sh"), content);
         
         // Creating container configuration
         var parameters = Configuration.ToInstallationCreateParameters(
@@ -49,6 +51,26 @@ public partial class Server
 
         var dockerClient = ServiceProvider.GetRequiredService<DockerClient>();
 
+        // Ensure we can actually spawn the container
+        
+        try
+        {
+            var existingContainer = await dockerClient.Containers.InspectContainerAsync(InstallationContainerName);
+            
+            // Perform automatic cleanup / restore
+            
+            if (existingContainer.State.Running)
+                await dockerClient.Containers.KillContainerAsync(existingContainer.ID, new());
+            
+            await dockerClient.Containers.RemoveContainerAsync(existingContainer.ID, new());
+        }
+        catch (DockerContainerNotFoundException)
+        {
+            // Ignored
+        }
+
+        // Spawn the container
+        
         var container = await dockerClient.Containers.CreateContainerAsync(parameters);
         InstallationContainerId = container.ID;
 
@@ -56,4 +78,40 @@ public partial class Server
 
         await dockerClient.Containers.StartContainerAsync(InstallationContainerId, new());
     }
+
+    private async Task InternalFinishInstall()
+    {
+        var dockerClient = ServiceProvider.GetRequiredService<DockerClient>();
+        
+        ContainerInspectResponse? container;
+
+        try
+        {
+            container = await dockerClient.Containers.InspectContainerAsync(InstallationContainerId, CancellationToken.None);
+        }
+        catch (DockerContainerNotFoundException)
+        {
+            container = null;
+        }
+        
+        if(container == null)
+            return;
+
+        var exitCode = container.State.ExitCode;
+
+        await LogToConsole($"Installation finished with exit code: {exitCode}");
+        
+        if (exitCode != 0)
+        {
+            // TODO: Report installation failure
+        }
+
+        await LogToConsole("Removing container");
+        //await dockerClient.Containers.RemoveContainerAsync(InstallationContainerId, new());
+        InstallationContainerId = null;
+        
+        await ResetTasks();
+
+        await RemoveInstallationVolume();
+    }
 }

MoonlightServers.Daemon/Abstractions/Server.Notify.cs

@@ -4,5 +4,6 @@ namespace MoonlightServers.Daemon.Abstractions;
 
 public partial class Server
 {
-    public async Task NotifyContainerDied() => await StateMachine.FireAsync(ServerTrigger.NotifyContainerDied);
+    public async Task NotifyRuntimeContainerDied() => await StateMachine.FireAsync(ServerTrigger.NotifyRuntimeContainerDied);
+    public async Task NotifyInstallationContainerDied() => await StateMachine.FireAsync(ServerTrigger.NotifyInstallationContainerDied);
 }

MoonlightServers.Daemon/Abstractions/Server.Stop.cs

@@ -10,4 +10,9 @@ public partial class Server
     {
         await Console.WriteToInput($"{Configuration.StopCommand}\n\r");
     }
+
+    private async Task InternalFinishStop()
+    {
+        await Destroy();
+    }
 }

MoonlightServers.Daemon/Abstractions/Server.Storage.cs

@@ -7,49 +7,83 @@ public partial class Server
 {
     private async Task<string> EnsureRuntimeVolume()
     {
-        var appConfiguration = ServiceProvider.GetRequiredService<AppConfiguration>();
+        var hostPath = GetRuntimeVolumePath();
+        
+        await LogToConsole("Creating storage");
 
+        // Create volume if missing
+        if (!Directory.Exists(hostPath))
+            Directory.CreateDirectory(hostPath);
+        
+        // TODO: Virtual disk
+        
+        return hostPath;
+    }
+    
+    private string GetRuntimeVolumePath()
+    {
+        var appConfiguration = ServiceProvider.GetRequiredService<AppConfiguration>();
+        
         var hostPath = PathBuilder.Dir(
             appConfiguration.Storage.Volumes,
             Configuration.Id.ToString()
         );
         
-        await LogToConsole("Creating storage");
-
-        // TODO: Virtual disk
-        
-        // Create volume if missing
-        if (!Directory.Exists(hostPath))
-            Directory.CreateDirectory(hostPath);
-
         if (hostPath.StartsWith("/"))
             return hostPath;
         else
             return PathBuilder.JoinPaths(Directory.GetCurrentDirectory(), hostPath);
+    }
+    
+    public async Task RemoveRuntimeVolume()
+    {
+        var hostPath = GetRuntimeVolumePath();
         
-        return hostPath;
+        await LogToConsole("Removing storage");
+
+        // Remove volume if existing
+        if (Directory.Exists(hostPath))
+            Directory.Delete(hostPath, true);
+        
+        // TODO: Virtual disk
     }
     
     private async Task<string> EnsureInstallationVolume()
     {
-        var appConfiguration = ServiceProvider.GetRequiredService<AppConfiguration>();
+        var hostPath = GetInstallationVolumePath();
-
-        var hostPath = PathBuilder.Dir(
-            appConfiguration.Storage.Install,
-            Configuration.Id.ToString()
-        );
         
         await LogToConsole("Creating installation storage");
 
         // Create volume if missing
         if (!Directory.Exists(hostPath))
             Directory.CreateDirectory(hostPath);
+        
+        return hostPath;
+    }
 
+    private string GetInstallationVolumePath()
+    {
+        var appConfiguration = ServiceProvider.GetRequiredService<AppConfiguration>();
+        
+        var hostPath = PathBuilder.Dir(
+            appConfiguration.Storage.Install,
+            Configuration.Id.ToString()
+        );
+        
         if (hostPath.StartsWith("/"))
             return hostPath;
         else
             return PathBuilder.JoinPaths(Directory.GetCurrentDirectory(), hostPath);
+    }
+
+    public async Task RemoveInstallationVolume()
+    {
+        var hostPath = GetInstallationVolumePath();
         
-        return hostPath;
+        await LogToConsole("Removing installation storage");
+
+        // Remove volume if existing
+        if (Directory.Exists(hostPath))
+            Directory.Delete(hostPath, true);
     }
 }

MoonlightServers.Daemon/Enums/ServerTrigger.cs

@@ -8,5 +8,6 @@ public enum ServerTrigger
     Kill = 3,
     Reinstall = 4,
     NotifyOnline = 5,
-    NotifyContainerDied = 6
+    NotifyRuntimeContainerDied = 6,
+    NotifyInstallationContainerDied = 7
 }

MoonlightServers.Daemon/Extensions/ServerConfigurationExtensions.cs

@@ -176,8 +176,6 @@ public static class ServerConfigurationExtensions
 
         parameters.Cmd = [installShell, "/mnt/install/install.sh"];
 
-        parameters.HostConfig.AutoRemove = true;
-
         return parameters;
     }
 

MoonlightServers.Daemon/Extensions/StateConfigurationExtensions.cs

@@ -0,0 +1,66 @@
+using Stateless;
+
+namespace MoonlightServers.Daemon.Extensions;
+
+public static class StateConfigurationExtensions
+{
+    public static StateMachine<TState, TTrigger>.StateConfiguration OnExitFrom<TState, TTrigger>(
+        this StateMachine<TState, TTrigger>.StateConfiguration configuration, TTrigger trigger, Action entryAction
+    )
+    {
+        configuration.OnExit(transition =>
+        {
+            if(!transition.Trigger!.Equals(trigger))
+                return;
+            
+            entryAction.Invoke();
+        });
+
+        return configuration;
+    }
+    
+    public static StateMachine<TState, TTrigger>.StateConfiguration OnExitFrom<TState, TTrigger>(
+        this StateMachine<TState, TTrigger>.StateConfiguration configuration, TTrigger trigger, Action<StateMachine<TState, TTrigger>.Transition> entryAction
+    )
+    {
+        configuration.OnExit(transition =>
+        {
+            if(!transition.Trigger!.Equals(trigger))
+                return;
+            
+            entryAction.Invoke(transition);
+        });
+
+        return configuration;
+    }
+    
+    public static StateMachine<TState, TTrigger>.StateConfiguration OnExitFromAsync<TState, TTrigger>(
+        this StateMachine<TState, TTrigger>.StateConfiguration configuration, TTrigger trigger, Func<Task> entryAction
+    )
+    {
+        configuration.OnExitAsync(transition =>
+        {
+            if(!transition.Trigger!.Equals(trigger))
+                return Task.CompletedTask;
+            
+            return entryAction.Invoke();
+        });
+
+        return configuration;
+    }
+    
+    public static StateMachine<TState, TTrigger>.StateConfiguration OnExitFromAsync<TState, TTrigger>(
+        this StateMachine<TState, TTrigger>.StateConfiguration configuration, TTrigger trigger, Func<StateMachine<TState, TTrigger>.Transition, Task> entryAction
+    )
+    {
+        configuration.OnExitAsync(transition =>
+        {
+            if(!transition.Trigger!.Equals(trigger))
+                return Task.CompletedTask;
+            
+            return entryAction.Invoke(transition);
+        });
+
+        return configuration;
+    }
+}

MoonlightServers.Daemon/Services/ServerService.cs

@@ -115,15 +115,27 @@ public class ServerService : IHostedLifecycleService
 
                             Server? server;
 
-                            lock (Servers)
-                                server = Servers.FirstOrDefault(x => x.RuntimeContainerId == message.ID || x.InstallationContainerId == message.ID);
-
                             // TODO: Maybe implement a lookup for containers which id isn't set in the cache
 
-                            if (server == null)
+                            // Check if it's a runtime container
-                                return;
+                            lock (Servers)
+                                server = Servers.FirstOrDefault(x => x.RuntimeContainerId == message.ID);
 
-                            await server.NotifyContainerDied();
+                            if (server != null)
+                            {
+                                await server.NotifyRuntimeContainerDied();
+                                return;
+                            }
+                            
+                            // Check if it's an installation container
+                            lock (Servers)
+                                server = Servers.FirstOrDefault(x => x.InstallationContainerId == message.ID);
+                            
+                            if (server != null)
+                            {
+                                await server.NotifyInstallationContainerDied();
+                                return;
+                            }
                         }), Cancellation.Token);
                 }
                 catch (TaskCanceledException)
@@ -210,10 +222,10 @@ public class ServerService : IHostedLifecycleService
     }
 
     public Task StartingAsync(CancellationToken cancellationToken)
-     => Task.CompletedTask;
+        => Task.CompletedTask;
 
     public Task StoppedAsync(CancellationToken cancellationToken)
-     => Task.CompletedTask;
+        => Task.CompletedTask;
 
     public async Task StoppingAsync(CancellationToken cancellationToken)
     {