Started implementing metrics system

Moonlight.ApiServer/Configuration/AppConfiguration.cs

@@ -11,6 +11,7 @@ public class AppConfiguration
     public DevelopmentConfig Development { get; set; } = new();
     public ClientConfig Client { get; set; } = new();
     public KestrelConfig Kestrel { get; set; } = new();
+    public MetricsData Metrics { get; set; } = new();
 
     public class ClientConfig
     {
@@ -59,4 +60,10 @@ public class AppConfiguration
         public int UploadLimit { get; set; } = 100;
         public string AllowedOrigins { get; set; } = "*";
     }
+    
+    public class MetricsData
+    {
+        public bool Enable { get; set; } = false;
+        public int Interval { get; set; } = 15;
+    }
 }

Moonlight.ApiServer/Implementations/Metrics/ApplicationMetric.cs

@@ -0,0 +1,35 @@
+using System.Diagnostics.Metrics;
+using Moonlight.ApiServer.Interfaces;
+using Moonlight.ApiServer.Services;
+
+namespace Moonlight.ApiServer.Implementations.Metrics;
+
+public class ApplicationMetric : IMetric
+{
+    private Gauge<long> MemoryUsage;
+    private Gauge<int> CpuUsage;
+    private Gauge<double> Uptime;
+    
+    public Task Initialize(Meter meter)
+    {
+        MemoryUsage = meter.CreateGauge<long>("moonlight_memory_usage");
+        CpuUsage = meter.CreateGauge<int>("moonlight_cpu_usage");
+        Uptime = meter.CreateGauge<double>("moonlight_uptime");
+        
+        return Task.CompletedTask;
+    }
+
+    public async Task Run(IServiceProvider provider, CancellationToken cancellationToken)
+    {
+        var applicationService = provider.GetRequiredService<ApplicationService>();
+
+        var memory = await applicationService.GetMemoryUsage();
+        MemoryUsage.Record(memory);
+
+        var uptime = await applicationService.GetUptime();
+        Uptime.Record(uptime.TotalSeconds);
+
+        var cpu = await applicationService.GetCpuUsage();
+        CpuUsage.Record(cpu);
+    }
+}

Moonlight.ApiServer/Implementations/Metrics/UsersMetric.cs

@@ -0,0 +1,27 @@
+using System.Diagnostics.Metrics;
+using Microsoft.EntityFrameworkCore;
+using MoonCore.Extended.Abstractions;
+using Moonlight.ApiServer.Database.Entities;
+using Moonlight.ApiServer.Interfaces;
+
+namespace Moonlight.ApiServer.Implementations.Metrics;
+
+public class UsersMetric : IMetric
+{
+    private Gauge<int> Users;
+    
+    public Task Initialize(Meter meter)
+    {
+        Users = meter.CreateGauge<int>("moonlight_users");
+        
+        return Task.CompletedTask;
+    }
+
+    public async Task Run(IServiceProvider provider, CancellationToken cancellationToken)
+    {
+        var usersRepo = provider.GetRequiredService<DatabaseRepository<User>>();
+        var count = await usersRepo.Get().CountAsync(cancellationToken: cancellationToken);
+        
+        Users.Record(count);
+    }
+}

Moonlight.ApiServer/Implementations/Startup/CoreStartup.cs

@@ -2,8 +2,12 @@ using Microsoft.OpenApi.Models;
 using Moonlight.ApiServer.Configuration;
 using Moonlight.ApiServer.Database;
 using Moonlight.ApiServer.Implementations.Diagnose;
+using Moonlight.ApiServer.Implementations.Metrics;
 using Moonlight.ApiServer.Interfaces;
 using Moonlight.ApiServer.Plugins;
+using Moonlight.ApiServer.Services;
+using OpenTelemetry.Metrics;
+using OpenTelemetry.Trace;
 
 namespace Moonlight.ApiServer.Implementations.Startup;
 
@@ -55,11 +59,42 @@ public class CoreStartup : IPluginStartup
 
         #endregion
 
+        #region Prometheus
+
+        if (configuration.Metrics.Enable)
+        {
+            builder.Services.AddSingleton<MetricsBackgroundService>();
+            builder.Services.AddHostedService(sp => sp.GetRequiredService<MetricsBackgroundService>());
+
+            builder.Services.AddSingleton<IMetric, ApplicationMetric>();
+            builder.Services.AddSingleton<IMetric, UsersMetric>();
+
+            builder.Services.AddOpenTelemetry()
+                .WithMetrics(providerBuilder =>
+                {
+                    providerBuilder.AddPrometheusExporter();
+                    providerBuilder.AddAspNetCoreInstrumentation();
+
+                    providerBuilder.AddMeter("moonlight");
+                });
+        }
+
+        #endregion
+
         return Task.CompletedTask;
     }
 
     public Task ConfigureApplication(IServiceProvider serviceProvider, IApplicationBuilder app)
     {
+        var configuration = serviceProvider.GetRequiredService<AppConfiguration>();
+        
+        #region Prometheus
+
+        if(configuration.Metrics.Enable)
+            app.UseOpenTelemetryPrometheusScrapingEndpoint();
+
+        #endregion
+        
         return Task.CompletedTask;
     }
 
@@ -67,7 +102,7 @@ public class CoreStartup : IPluginStartup
     {
         var configuration = serviceProvider.GetRequiredService<AppConfiguration>();
 
-        if(configuration.Development.EnableApiDocs)
+        if (configuration.Development.EnableApiDocs)
             routeBuilder.MapSwagger("/api/swagger/{documentName}");
 
         return Task.CompletedTask;

Moonlight.ApiServer/Interfaces/IMetric.cs

@@ -0,0 +1,9 @@
+using System.Diagnostics.Metrics;
+
+namespace Moonlight.ApiServer.Interfaces;
+
+public interface IMetric
+{
+    public Task Initialize(Meter meter);
+    public Task Run(IServiceProvider provider, CancellationToken cancellationToken);
+}

Moonlight.ApiServer/Moonlight.ApiServer.csproj

@@ -11,6 +11,7 @@
   </ItemGroup>
   <ItemGroup>
     <Folder Include="Database\Migrations\" />
+    <Folder Include="Helpers\" />
   </ItemGroup>
   <ItemGroup>
     <Content Include="..\.dockerignore">
@@ -36,6 +37,9 @@
     <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="9.0.5" />
     <PackageReference Include="MoonCore" Version="1.8.8" />
     <PackageReference Include="MoonCore.Extended" Version="1.3.4" />
+    <PackageReference Include="OpenTelemetry.Exporter.Prometheus.AspNetCore" Version="1.12.0-beta.1" />
+    <PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.12.0" />
+    <PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.12.0" />
     <PackageReference Include="SharpZipLib" Version="1.4.2" />
     <PackageReference Include="Swashbuckle.AspNetCore" Version="8.1.1" />
     <PackageReference Include="Ben.Demystifier" Version="0.4.1" />

Moonlight.ApiServer/Services/MetricsBackgroundService.cs

@@ -0,0 +1,78 @@
+using System.Diagnostics.Metrics;
+using Moonlight.ApiServer.Configuration;
+using Moonlight.ApiServer.Interfaces;
+
+namespace Moonlight.ApiServer.Services;
+
+public class MetricsBackgroundService : BackgroundService
+{
+    private readonly ILogger<MetricsBackgroundService> Logger;
+    private readonly IServiceProvider ServiceProvider;
+    private readonly AppConfiguration Configuration;
+
+    private readonly IMetric[] Metrics;
+    private readonly Meter Meter;
+
+    public MetricsBackgroundService(
+        IServiceProvider serviceProvider,
+        IMeterFactory meterFactory,
+        IEnumerable<IMetric> metrics,
+        ILogger<MetricsBackgroundService> logger,
+        AppConfiguration configuration
+    )
+    {
+        ServiceProvider = serviceProvider;
+        Logger = logger;
+        Configuration = configuration;
+
+        Meter = meterFactory.Create("moonlight");
+
+        Metrics = metrics.ToArray();
+    }
+
+    private async Task Initialize()
+    {
+        Logger.LogDebug(
+            "Initializing metrics: {names}",
+            string.Join(", ", Metrics.Select(x => x.GetType().FullName))
+        );
+
+        foreach (var metric in Metrics)
+            await metric.Initialize(Meter);
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        await Initialize();
+
+        while (!stoppingToken.IsCancellationRequested)
+        {
+            using var scope = ServiceProvider.CreateScope();
+
+            foreach (var metric in Metrics)
+            {
+                try
+                {
+                    await metric.Run(scope.ServiceProvider, stoppingToken);
+                }
+                catch (TaskCanceledException)
+                {
+                    // Ignored
+                }
+                catch (Exception e)
+                {
+                    Logger.LogError(
+                        "An unhandled error occured while collecting metric {name}: {e}",
+                        metric.GetType().FullName,
+                        e
+                    );
+                }
+            }
+
+            await Task.Delay(
+                TimeSpan.FromSeconds(Configuration.Metrics.Interval),
+                stoppingToken
+            );
+        }
+    }
+}