Added base health check and diagnostic system

Moonlight/App/Diagnostics/HealthChecks/DatabaseHealthCheck.cs

@@ -0,0 +1,32 @@
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+using Moonlight.App.Database;
+
+namespace Moonlight.App.Diagnostics.HealthChecks;
+
+public class DatabaseHealthCheck : IHealthCheck
+{
+    private readonly DataContext DataContext;
+
+    public DatabaseHealthCheck(DataContext dataContext)
+    {
+        DataContext = dataContext;
+    }
+
+    public async Task<HealthCheckResult> CheckHealthAsync(
+        HealthCheckContext context,
+        CancellationToken cancellationToken = new CancellationToken())
+    {
+        try
+        {
+            await DataContext.Database.OpenConnectionAsync(cancellationToken);
+            await DataContext.Database.CloseConnectionAsync();
+            
+            return HealthCheckResult.Healthy("Database is online");
+        }
+        catch (Exception e)
+        {
+            return HealthCheckResult.Unhealthy("Database is offline", e);
+        }
+    }
+}

Moonlight/App/Diagnostics/HealthChecks/NodeHealthCheck.cs

@@ -0,0 +1,58 @@
+using System.Diagnostics;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+using Moonlight.App.Database.Entities;
+using Moonlight.App.Repositories;
+using Moonlight.App.Services;
+
+namespace Moonlight.App.Diagnostics.HealthChecks;
+
+public class NodeHealthCheck : IHealthCheck
+{
+    private readonly Repository<Node> NodeRepository;
+    private readonly NodeService NodeService;
+
+    public NodeHealthCheck(Repository<Node> nodeRepository, NodeService nodeService)
+    {
+        NodeRepository = nodeRepository;
+        NodeService = nodeService;
+    }
+
+    public async Task<HealthCheckResult> CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = new CancellationToken())
+    {
+        var nodes = NodeRepository.Get().ToArray();
+        
+        var results = new Dictionary<Node, bool>();
+        var healthCheckData = new Dictionary<string, object>();
+        
+        foreach (var node in nodes)
+        {
+            try
+            {
+                await NodeService.GetStatus(node);
+                
+                results.Add(node, true);
+            }
+            catch (Exception e)
+            {
+                results.Add(node, false);
+                healthCheckData.Add(node.Name, e.ToStringDemystified());
+            }
+        }
+
+        var offlineNodes = results
+            .Where(x => !x.Value)
+            .ToArray();
+
+        if (offlineNodes.Length == nodes.Length)
+        {
+            return HealthCheckResult.Unhealthy("All nodes are offline", null, healthCheckData);
+        }
+        
+        if (offlineNodes.Length == 0)
+        {
+            return HealthCheckResult.Healthy("All nodes are online");
+        }
+            
+        return HealthCheckResult.Degraded($"{offlineNodes.Length} nodes are offline", null, healthCheckData);
+    }
+}

Moonlight/App/Models/Misc/HealthCheck.cs

@@ -0,0 +1,19 @@
+using Newtonsoft.Json;
+
+namespace Moonlight.App.Models.Misc;
+
+public class HealthCheck
+{
+    public string Status { get; set; }
+    public TimeSpan TotalDuration { get; set; }
+    public Dictionary<string, HealthCheckEntry> Entries { get; set; } = new();
+    
+    public class HealthCheckEntry
+    {
+        public Dictionary<string, string> Data { get; set; } = new();
+        public string Description { get; set; }
+        public TimeSpan Duration { get; set; }
+        public string Status { get; set; }
+        public List<string> Tags { get; set; } = new();
+    }
+}

Moonlight/Moonlight.csproj

@@ -9,6 +9,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="AspNetCore.HealthChecks.UI.Client" Version="6.0.5" />
     <PackageReference Include="Blazor-ApexCharts" Version="0.9.16-beta" />
     <PackageReference Include="BCrypt.Net-Next" Version="4.0.3" />
     <PackageReference Include="Ben.Demystifier" Version="0.4.1" />

Moonlight/Program.cs

@@ -1,12 +1,14 @@
 using BlazorDownloadFile;
 using BlazorTable;
 using CurrieTechnologies.Razor.SweetAlert2;
+using HealthChecks.UI.Client;
 using Logging.Net;
 using Moonlight.App.ApiClients.CloudPanel;
 using Moonlight.App.ApiClients.Daemon;
 using Moonlight.App.ApiClients.Paper;
 using Moonlight.App.ApiClients.Wings;
 using Moonlight.App.Database;
+using Moonlight.App.Diagnostics.HealthChecks;
 using Moonlight.App.Events;
 using Moonlight.App.Helpers;
 using Moonlight.App.Helpers.Wings;
@@ -66,6 +68,9 @@ namespace Moonlight
                     options.HandshakeTimeout = TimeSpan.FromSeconds(10);
                 });
             builder.Services.AddHttpContextAccessor();
+            builder.Services.AddHealthChecks()
+                .AddCheck<DatabaseHealthCheck>("Database")
+                .AddCheck<NodeHealthCheck>("Nodes");
 
             // Databases
             builder.Services.AddDbContext<DataContext>();
@@ -186,6 +191,10 @@ namespace Moonlight
 
             app.MapBlazorHub();
             app.MapFallbackToPage("/_Host");
+            app.MapHealthChecks("/_health", new()
+            {
+                ResponseWriter = UIResponseWriter.WriteHealthCheckUIResponse
+            });
 
             // AutoStart services
             _ = app.Services.GetRequiredService<CleanupService>();

Moonlight/Shared/Components/Partials/HealthCheckView.razor

@@ -0,0 +1,59 @@
+@using Moonlight.App.Models.Misc
+@using System.Text
+@using Moonlight.App.Helpers
+
+@{
+    string GetStatusColor(string s)
+    {
+        if (s == "Healthy")
+            return "success";
+        else if (s == "Unhealthy")
+            return "danger";
+        else
+            return "warning";
+    }
+}
+
+<div class="card">
+    <div class="card-header">
+        <div class="card-title">
+            <TL>Moonlight health</TL>:
+            <div class="ps-3 text-@(GetStatusColor(HealthCheck.Status))">
+                <TL>@(HealthCheck.Status)</TL>
+            </div>
+        </div>
+    </div>
+    <div class="card-body">
+        <div class="accordion" id="healthCheck">
+            @foreach (var entry in HealthCheck.Entries)
+            {
+                <div class="accordion-item">
+                    <h2 class="accordion-header" id="healthCheck_1_header_@(entry.Key.ToLower())">
+                        <button class="accordion-button fs-4 fw-semibold text-@(GetStatusColor(entry.Value.Status))" type="button" data-bs-toggle="collapse" data-bs-target="#healthCheck_body_@(entry.Key.ToLower())">
+                            @(entry.Key)
+                        </button>
+                    </h2>
+                    <div id="healthCheck_body_@(entry.Key.ToLower())" class="accordion-collapse collapse show" data-bs-parent="#healthCheck">
+                        <div class="accordion-body">
+                            <b><TL>Status</TL>:</b>&nbsp;<TL>@(entry.Value.Status)</TL><br/>
+                            <b><TL>Description</TL>:</b>&nbsp;@(entry.Value.Description)<br/>
+                            <br/>
+                            @foreach (var x in entry.Value.Data)
+                            {
+                                <b>@(x.Key)</b>
+                                <br/>
+                                @(x.Value)<br/>
+                            }
+                        </div>
+                    </div>
+                </div>
+            }
+        </div>
+    </div>
+</div>
+
+@code
+{
+    [Parameter]
+    public HealthCheck HealthCheck { get; set; }
+}

Moonlight/Shared/Views/Admin/Index.razor

@@ -3,11 +3,15 @@
 @using Moonlight.App.Repositories
 @using Moonlight.App.Repositories.Domains
 @using Moonlight.App.Database.Entities
+@using Moonlight.App.Models.Misc
+@using Moonlight.App.Services
+@using Newtonsoft.Json
 
 @inject ServerRepository ServerRepository
 @inject UserRepository UserRepository
 @inject Repository<WebSpace> WebSpaceRepository
 @inject DomainRepository DomainRepository
+@inject ConfigService ConfigService
 
 <OnlyAdmin>
     <LazyLoader Load="Load">
@@ -97,6 +101,7 @@
                 </a>
             </div>
         </div>
+        <HealthCheckView HealthCheck="@HealthCheckData" />
     </LazyLoader>
 </OnlyAdmin>
 
@@ -107,13 +112,33 @@
     private int DomainCount = 0;
     private int WebSpaceCount = 0;
 
-    private Task Load(LazyLoader lazyLoader)
+    private HealthCheck HealthCheckData;
+
+    private async Task Load(LazyLoader lazyLoader)
     {
         ServerCount = ServerRepository.Get().Count();
         UserCount = UserRepository.Get().Count();
         DomainCount = DomainRepository.Get().Count();
         WebSpaceCount = WebSpaceRepository.Get().Count();
 
-        return Task.CompletedTask;
+        var appUrl = ConfigService
+            .GetSection("Moonlight")
+            .GetValue<string>("AppUrl");
+
+        try
+        {
+            using var client = new HttpClient();
+            var json = await client.GetStringAsync($"{appUrl}/_health");
+            HealthCheckData = JsonConvert.DeserializeObject<HealthCheck>(json) ?? new();
+        }
+        catch (Exception)
+        {
+            HealthCheckData = new()
+            {
+                Status = "Healthy",
+                Entries = new(),
+                TotalDuration = TimeSpan.MinValue
+            };
+        }
     }
 }