Merge pull request #150 from Moonlight-Panel/AddHealthChecks

Add health checks

Moonlight/App/Diagnostics/HealthChecks/DaemonHealthCheck.cs

@@ -0,0 +1,58 @@
+using System.Diagnostics;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+using Moonlight.App.Database.Entities;
+using Moonlight.App.Repositories;
+using Moonlight.App.Services;
+
+namespace Moonlight.App.Diagnostics.HealthChecks;
+
+public class DaemonHealthCheck : IHealthCheck
+{
+    private readonly Repository<Node> NodeRepository;
+    private readonly NodeService NodeService;
+
+    public DaemonHealthCheck(Repository<Node> nodeRepository, NodeService nodeService)
+    {
+        NodeRepository = nodeRepository;
+        NodeService = nodeService;
+    }
+
+    public async Task<HealthCheckResult> CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = new CancellationToken())
+    {
+        var nodes = NodeRepository.Get().ToArray();
+        
+        var results = new Dictionary<Node, bool>();
+        var healthCheckData = new Dictionary<string, object>();
+        
+        foreach (var node in nodes)
+        {
+            try
+            {
+                await NodeService.GetCpuMetrics(node);
+                
+                results.Add(node, true);
+            }
+            catch (Exception e)
+            {
+                results.Add(node, false);
+                healthCheckData.Add(node.Name, e.ToStringDemystified());
+            }
+        }
+
+        var offlineNodes = results
+            .Where(x => !x.Value)
+            .ToArray();
+
+        if (offlineNodes.Length == nodes.Length)
+        {
+            return HealthCheckResult.Unhealthy("All node daemons are offline", null, healthCheckData);
+        }
+        
+        if (offlineNodes.Length == 0)
+        {
+            return HealthCheckResult.Healthy("All node daemons are online");
+        }
+            
+        return HealthCheckResult.Degraded($"{offlineNodes.Length} node daemons are offline", null, healthCheckData);
+    }
+}

Moonlight/App/Diagnostics/HealthChecks/DatabaseHealthCheck.cs

@@ -0,0 +1,32 @@
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+using Moonlight.App.Database;
+
+namespace Moonlight.App.Diagnostics.HealthChecks;
+
+public class DatabaseHealthCheck : IHealthCheck
+{
+    private readonly DataContext DataContext;
+
+    public DatabaseHealthCheck(DataContext dataContext)
+    {
+        DataContext = dataContext;
+    }
+
+    public async Task<HealthCheckResult> CheckHealthAsync(
+        HealthCheckContext context,
+        CancellationToken cancellationToken = new CancellationToken())
+    {
+        try
+        {
+            await DataContext.Database.OpenConnectionAsync(cancellationToken);
+            await DataContext.Database.CloseConnectionAsync();
+            
+            return HealthCheckResult.Healthy("Database is online");
+        }
+        catch (Exception e)
+        {
+            return HealthCheckResult.Unhealthy("Database is offline", e);
+        }
+    }
+}

Moonlight/App/Diagnostics/HealthChecks/NodeHealthCheck.cs

@@ -0,0 +1,58 @@
+using System.Diagnostics;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+using Moonlight.App.Database.Entities;
+using Moonlight.App.Repositories;
+using Moonlight.App.Services;
+
+namespace Moonlight.App.Diagnostics.HealthChecks;
+
+public class NodeHealthCheck : IHealthCheck
+{
+    private readonly Repository<Node> NodeRepository;
+    private readonly NodeService NodeService;
+
+    public NodeHealthCheck(Repository<Node> nodeRepository, NodeService nodeService)
+    {
+        NodeRepository = nodeRepository;
+        NodeService = nodeService;
+    }
+
+    public async Task<HealthCheckResult> CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = new CancellationToken())
+    {
+        var nodes = NodeRepository.Get().ToArray();
+        
+        var results = new Dictionary<Node, bool>();
+        var healthCheckData = new Dictionary<string, object>();
+        
+        foreach (var node in nodes)
+        {
+            try
+            {
+                await NodeService.GetStatus(node);
+                
+                results.Add(node, true);
+            }
+            catch (Exception e)
+            {
+                results.Add(node, false);
+                healthCheckData.Add(node.Name, e.ToStringDemystified());
+            }
+        }
+
+        var offlineNodes = results
+            .Where(x => !x.Value)
+            .ToArray();
+
+        if (offlineNodes.Length == nodes.Length)
+        {
+            return HealthCheckResult.Unhealthy("All nodes are offline", null, healthCheckData);
+        }
+        
+        if (offlineNodes.Length == 0)
+        {
+            return HealthCheckResult.Healthy("All nodes are online");
+        }
+            
+        return HealthCheckResult.Degraded($"{offlineNodes.Length} nodes are offline", null, healthCheckData);
+    }
+}

Moonlight/App/Models/Misc/HealthCheck.cs

@@ -0,0 +1,19 @@
+using Newtonsoft.Json;
+
+namespace Moonlight.App.Models.Misc;
+
+public class HealthCheck
+{
+    public string Status { get; set; }
+    public TimeSpan TotalDuration { get; set; }
+    public Dictionary<string, HealthCheckEntry> Entries { get; set; } = new();
+    
+    public class HealthCheckEntry
+    {
+        public Dictionary<string, string> Data { get; set; } = new();
+        public string Description { get; set; }
+        public TimeSpan Duration { get; set; }
+        public string Status { get; set; }
+        public List<string> Tags { get; set; } = new();
+    }
+}

Moonlight/Moonlight.csproj

@@ -9,6 +9,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="AspNetCore.HealthChecks.UI.Client" Version="6.0.5" />
     <PackageReference Include="Blazor-ApexCharts" Version="0.9.16-beta" />
     <PackageReference Include="BCrypt.Net-Next" Version="4.0.3" />
     <PackageReference Include="Ben.Demystifier" Version="0.4.1" />

Moonlight/Program.cs

@@ -1,12 +1,14 @@
 using BlazorDownloadFile;
 using BlazorTable;
 using CurrieTechnologies.Razor.SweetAlert2;
+using HealthChecks.UI.Client;
 using Logging.Net;
 using Moonlight.App.ApiClients.CloudPanel;
 using Moonlight.App.ApiClients.Daemon;
 using Moonlight.App.ApiClients.Paper;
 using Moonlight.App.ApiClients.Wings;
 using Moonlight.App.Database;
+using Moonlight.App.Diagnostics.HealthChecks;
 using Moonlight.App.Events;
 using Moonlight.App.Helpers;
 using Moonlight.App.Helpers.Wings;
@@ -66,6 +68,10 @@ namespace Moonlight
                     options.HandshakeTimeout = TimeSpan.FromSeconds(10);
                 });
             builder.Services.AddHttpContextAccessor();
+            builder.Services.AddHealthChecks()
+                .AddCheck<DatabaseHealthCheck>("Database")
+                .AddCheck<NodeHealthCheck>("Nodes")
+                .AddCheck<DaemonHealthCheck>("Daemons");
 
             // Databases
             builder.Services.AddDbContext<DataContext>();
@@ -186,6 +192,10 @@ namespace Moonlight
 
             app.MapBlazorHub();
             app.MapFallbackToPage("/_Host");
+            app.MapHealthChecks("/_health", new()
+            {
+                ResponseWriter = UIResponseWriter.WriteHealthCheckUIResponse
+            });
 
             // AutoStart services
             _ = app.Services.GetRequiredService<CleanupService>();

Moonlight/Shared/Components/Partials/HealthCheckView.razor

@@ -0,0 +1,59 @@
+@using Moonlight.App.Models.Misc
+@using System.Text
+@using Moonlight.App.Helpers
+
+@{
+    string GetStatusColor(string s)
+    {
+        if (s == "Healthy")
+            return "success";
+        else if (s == "Unhealthy")
+            return "danger";
+        else
+            return "warning";
+    }
+}
+
+<div class="card">
+    <div class="card-header">
+        <div class="card-title">
+            <TL>Moonlight health</TL>:
+            <div class="ps-3 text-@(GetStatusColor(HealthCheck.Status))">
+                <TL>@(HealthCheck.Status)</TL>
+            </div>
+        </div>
+    </div>
+    <div class="card-body">
+        <div class="accordion" id="healthCheck">
+            @foreach (var entry in HealthCheck.Entries)
+            {
+                <div class="accordion-item">
+                    <h2 class="accordion-header" id="healthCheck_1_header_@(entry.Key.ToLower())">
+                        <button class="accordion-button fs-4 fw-semibold text-@(GetStatusColor(entry.Value.Status))" type="button" data-bs-toggle="collapse" data-bs-target="#healthCheck_body_@(entry.Key.ToLower())">
+                            @(entry.Key)
+                        </button>
+                    </h2>
+                    <div id="healthCheck_body_@(entry.Key.ToLower())" class="accordion-collapse collapse" data-bs-parent="#healthCheck">
+                        <div class="accordion-body">
+                            <b><TL>Status</TL>:</b>&nbsp;<TL>@(entry.Value.Status)</TL><br/>
+                            <b><TL>Description</TL>:</b>&nbsp;@(entry.Value.Description)<br/>
+                            <br/>
+                            @foreach (var x in entry.Value.Data)
+                            {
+                                <b>@(x.Key)</b>
+                                <br/>
+                                @(x.Value)<br/>
+                            }
+                        </div>
+                    </div>
+                </div>
+            }
+        </div>
+    </div>
+</div>
+
+@code
+{
+    [Parameter]
+    public HealthCheck HealthCheck { get; set; }
+}

Moonlight/Shared/Views/Admin/Index.razor

@@ -3,11 +3,16 @@
 @using Moonlight.App.Repositories
 @using Moonlight.App.Repositories.Domains
 @using Moonlight.App.Database.Entities
+@using Moonlight.App.Models.Misc
+@using Moonlight.App.Services
+@using Newtonsoft.Json
+@using Logging.Net
 
 @inject ServerRepository ServerRepository
 @inject UserRepository UserRepository
 @inject Repository<WebSpace> WebSpaceRepository
 @inject DomainRepository DomainRepository
+@inject ConfigService ConfigService
 
 <OnlyAdmin>
     <LazyLoader Load="Load">
@@ -97,6 +102,28 @@
                 </a>
             </div>
         </div>
+
+        <LazyLoader Load="LoadHealthCheckData">
+            @if (HealthCheckData == null)
+            {
+                <div class="card">
+                    <div class="card-header">
+                        <div class="card-title">
+                            <TL>Moonlight health</TL>
+                        </div>
+                    </div>
+                    <div class="card-body">
+                        <div class="alert alert-warning">
+                            <TL>Unable to fetch health check data</TL>
+                        </div>
+                    </div>
+                </div>
+            }
+            else
+            {
+                <HealthCheckView HealthCheck="@HealthCheckData"/>
+            }
+        </LazyLoader>
     </LazyLoader>
 </OnlyAdmin>
 
@@ -107,6 +134,8 @@
     private int DomainCount = 0;
     private int WebSpaceCount = 0;
 
+    private HealthCheck? HealthCheckData;
+
     private Task Load(LazyLoader lazyLoader)
     {
         ServerCount = ServerRepository.Get().Count();
@@ -116,4 +145,26 @@
 
         return Task.CompletedTask;
     }
+
+    private async Task LoadHealthCheckData(LazyLoader lazyLoader)
+    {
+        await lazyLoader.SetText("Loading health check data");
+
+        var appUrl = ConfigService
+            .GetSection("Moonlight")
+            .GetValue<string>("AppUrl");
+
+        try
+        {
+            using var client = new HttpClient();
+            var json = await client.GetStringAsync($"{appUrl}/_health");
+            HealthCheckData = JsonConvert.DeserializeObject<HealthCheck>(json) ?? new();
+        }
+        catch (Exception e)
+        {
+            HealthCheckData = null;
+            Logger.Warn("Unable to fetch health check data");
+            Logger.Warn(e);
+        }
+    }
 }