Moonlight/Moonlight/App/Services/Background/CleanupService.cs

using Microsoft.EntityFrameworkCore;
using MineStatLib;
using Moonlight.App.ApiClients.Daemon.Resources;
using Moonlight.App.ApiClients.Wings;
using Moonlight.App.Database.Entities;
using Moonlight.App.Events;
using Moonlight.App.Helpers;
using Moonlight.App.Repositories;
using Moonlight.App.Repositories.Servers;
using Newtonsoft.Json;

namespace Moonlight.App.Services.Background;

public class CleanupService
{
    #region Stats
    public DateTime StartedAt { get; private set; }
    public DateTime CompletedAt { get; private set; }
    public int ServersCleaned { get; private set; }
    public int CleanupsPerformed { get; private set; }
    public int ServersRunning { get; private set; }
    public bool IsRunning { get; private set; }
    #endregion
    
    private readonly ConfigService ConfigService;
    private readonly DateTimeService DateTimeService;
    private readonly EventSystem Event;
    private readonly IServiceScopeFactory ServiceScopeFactory;
    private readonly PeriodicTimer Timer;

    public CleanupService(
        ConfigService configService,
        IServiceScopeFactory serviceScopeFactory,
        DateTimeService dateTimeService,
        EventSystem eventSystem)
    {
        ServiceScopeFactory = serviceScopeFactory;
        DateTimeService = dateTimeService;
        ConfigService = configService;
        Event = eventSystem;
        
        StartedAt = DateTimeService.GetCurrent();
        CompletedAt = DateTimeService.GetCurrent();
        IsRunning = false;

        var config = ConfigService.Get().Moonlight.Cleanup;

        if (!config.Enable || ConfigService.DebugMode)
        {
            Logger.Info("Disabling cleanup service");
            return;
        }
        
        Timer = new(TimeSpan.FromMinutes(config.Wait));

        Task.Run(Run);
    }

    private async Task Run()
    {
        while (await Timer.WaitForNextTickAsync())
        {
            IsRunning = true;
            
            using var scope = ServiceScopeFactory.CreateScope();
            var config = ConfigService.Get().Moonlight.Cleanup;

            var maxCpu = config.Cpu;
            var minMemory = config.Memory;
            var maxUptime = config.Uptime;
            var minUptime = config.MinUptime;

            var nodeRepository = scope.ServiceProvider.GetRequiredService<NodeRepository>();
            var nodeService = scope.ServiceProvider.GetRequiredService<NodeService>();

            var nodes = nodeRepository
                .Get()
                .ToArray();

            foreach (var node in nodes)
            {
                try
                {
                    var cpuMetrics = await nodeService.GetCpuMetrics(node);
                    var memoryMetrics = await nodeService.GetMemoryMetrics(node);

                    bool executeCleanup;
                    
                    if (cpuMetrics.CpuUsage > maxCpu)
                    {
                        Logger.Debug($"{node.Name}: CPU Usage is too high");
                        Logger.Debug($"Usage:  {cpuMetrics.CpuUsage}");
                        Logger.Debug($"Max CPU:  {maxCpu}");
                        executeCleanup = true;
                    }
                    else if (Formatter.BytesToGb(memoryMetrics.Total) - Formatter.BytesToGb(memoryMetrics.Used) <
                             minMemory / 1024D)
                    {
                        Logger.Debug($"{node.Name}: Memory Usage is too high");
                        Logger.Debug($"Memory (Total):  {Formatter.BytesToGb(memoryMetrics.Total)}");
                        Logger.Debug($"Memory (Used):  {Formatter.BytesToGb(memoryMetrics.Used)}");
                        Logger.Debug(
                            $"Memory (Free):  {Formatter.BytesToGb(memoryMetrics.Total) - Formatter.BytesToGb(memoryMetrics.Used)}");
                        Logger.Debug($"Min Memory:  {minMemory}");
                        executeCleanup = true;
                    }
                    else
                        executeCleanup = false;
                    
                    if (executeCleanup)
                    {
                        var dockerMetrics = await nodeService.GetDockerMetrics(node);

                        var serverRepository = scope.ServiceProvider.GetRequiredService<ServerRepository>();
                        var imageRepository = scope.ServiceProvider.GetRequiredService<ImageRepository>();

                        var images = imageRepository
                            .Get()
                            .ToArray();

                        var imagesWithFlag = images
                            .Where(x => 
                                (JsonConvert.DeserializeObject<string[]>(x.TagsJson) ?? Array.Empty<string>()).Contains("cleanup")
                            )
                            .ToArray();
                        
                        var containerMappedToServers = new Dictionary<Container, Server>();

                        foreach (var container in dockerMetrics.Containers)
                        {
                            if (Guid.TryParse(container.Name, out Guid uuid))
                            {
                                var server = serverRepository
                                    .Get()
                                    .Include(x => x.Image)
                                    .Include(x => x.MainAllocation)
                                    .Include(x => x.Variables)
                                    .FirstOrDefault(x => x.Uuid == uuid);

                                if (server != null && imagesWithFlag.Any(y => y.Id == server.Image.Id))
                                {
                                    containerMappedToServers.Add(container, server);
                                }
                            }
                        }

                        var serverService = scope.ServiceProvider.GetRequiredService<ServerService>();
                        
                        foreach (var containerMapped in containerMappedToServers)
                        {
                            var server = containerMapped.Value;
                            
                            try
                            {
                                var stats = await serverService.GetDetails(server);

                                if (server.IsCleanupException)
                                {
                                    if (stats.Utilization.Uptime > TimeSpan.FromHours(maxUptime).TotalMilliseconds)
                                    {
                                        var players = GetPlayers(node, server.MainAllocation);

                                        if (players == 0)
                                        {
                                            Logger.Debug($"Restarted {server.Name} ({server.Uuid}) on node {node.Name}");
                                            await serverService.SetPowerState(server, PowerSignal.Restart);
                                            
                                            ServersCleaned++;
                                        }
                                        else
                                        {
                                            ServersRunning++;
                                        }
                                        
                                        await Event.Emit("cleanup.updated");
                                    }
                                }
                                else
                                {
                                    if (stats.Utilization.Uptime > TimeSpan.FromMinutes(minUptime).TotalMilliseconds)
                                    {
                                        var players = GetPlayers(node, server.MainAllocation);

                                        if (players < 1)
                                        {
                                            var j2SVar = server.Variables.FirstOrDefault(x => x.Key == "J2S");
                                            var handleJ2S = j2SVar != null && j2SVar.Value == "1";

                                            if (handleJ2S)
                                            {
                                                Logger.Debug($"Restarted (cleanup) {server.Name} ({server.Uuid}) on node {node.Name}");
                                                await serverService.SetPowerState(server, PowerSignal.Restart);
                                            }
                                            else
                                            {
                                                Logger.Debug($"Stopped {server.Name} ({server.Uuid}) on node {node.Name}");
                                                await serverService.SetPowerState(server, PowerSignal.Stop);
                                            }

                                            ServersCleaned++;
                                        }
                                        else
                                        {
                                            ServersRunning++;
                                        }
                                        
                                        await Event.Emit("cleanup.updated");
                                    }
                                }
                            }
                            catch (Exception e)
                            {
                                Logger.Warn($"Error checking server {server.Name} ({server.Id})");
                                Logger.Warn(e);
                            }
                        }
                    }
                }
                catch (Exception e)
                {
                    Logger.Error($"Error performing cleanup on node {node.Name} ({node.Id})");
                    Logger.Error(e);
                }
            }

            IsRunning = false;
            CleanupsPerformed++;
            await Event.Emit("cleanup.updated");
        }
    }

    private int GetPlayers(Node node, NodeAllocation allocation)
    {
        var ms = new MineStat(node.Fqdn, (ushort)allocation.Port);
        
        //TODO: Add fake player check

        if (ms.ServerUp)
        {
            return ms.CurrentPlayersInt;
        }
        
        return -1;
    }
}