Finished compile time plugin loading. Refactored plugin loading. Extended build helper script

This commit is contained in:
2025-05-13 20:48:50 +02:00
parent 8126250d1a
commit a579dd4759
28 changed files with 1169 additions and 741 deletions

View File

@@ -0,0 +1,370 @@
using System.IO.Compression;
using System.Xml.Linq;
using Cocona;
using Scripts.Helpers;
namespace Scripts.Commands;
public class PackCommand
{
private readonly CommandHelper CommandHelper;
private readonly string TmpDir = "/tmp/mlbuild";
public PackCommand(CommandHelper commandHelper)
{
CommandHelper = commandHelper;
}
[Command("pack", Description = "Packs the specified folder/solution into nuget packages")]
public async Task Pack(
[Argument] string solutionDirectory,
[Argument] string outputLocation,
[Option] string buildConfiguration = "Debug"
)
{
if (!Directory.Exists(solutionDirectory))
{
Console.WriteLine("The specified solution directory does not exist");
return;
}
if (!Directory.Exists(outputLocation))
Directory.CreateDirectory(outputLocation);
if (Directory.Exists(TmpDir))
Directory.Delete(TmpDir, true);
Directory.CreateDirectory(TmpDir);
// Find the project files
Console.WriteLine("Searching for projects inside the specified folder");
var csProjFiles = Directory.GetFiles(solutionDirectory, "*csproj", SearchOption.AllDirectories);
// Show the user
Console.WriteLine($"Found {csProjFiles.Length} project(s) to check:");
foreach (var csProjFile in csProjFiles)
Console.WriteLine($"- {Path.GetFullPath(csProjFile)}");
// Filter out project files which have specific tags specified
Console.WriteLine("Filtering projects by tags");
List<string> apiServerProjects = [];
List<string> frontendProjects = [];
List<string> sharedProjects = [];
foreach (var csProjFile in csProjFiles)
{
await using var fs = File.Open(
csProjFile,
FileMode.Open,
FileAccess.ReadWrite,
FileShare.ReadWrite
);
var document = await XDocument.LoadAsync(fs, LoadOptions.None, CancellationToken.None);
fs.Close();
// Search for tag definitions
var packageTagsElements = document.Descendants("PackageTags").ToArray();
if (packageTagsElements.Length == 0)
{
Console.WriteLine($"No package tags found in {Path.GetFullPath(csProjFile)}. Skipping it when packing");
continue;
}
var packageTags = packageTagsElements.First().Value;
if (packageTags.Contains("apiserver", StringComparison.InvariantCultureIgnoreCase))
apiServerProjects.Add(csProjFile);
if (packageTags.Contains("frontend", StringComparison.InvariantCultureIgnoreCase))
frontendProjects.Add(csProjFile);
if (packageTags.Contains("shared", StringComparison.InvariantCultureIgnoreCase))
sharedProjects.Add(csProjFile);
}
Console.WriteLine(
$"Found {apiServerProjects.Count} api server project(s), {frontendProjects.Count} frontend project(s) and {sharedProjects.Count} shared project(s)");
// Now build all these projects so we can pack them
Console.WriteLine("Building and packing api server project(s)");
foreach (var apiServerProject in apiServerProjects)
{
await BuildProject(
apiServerProject,
buildConfiguration
);
var nugetFilePath = await PackProject(
apiServerProject,
TmpDir,
buildConfiguration
);
var nugetPackage = ZipFile.Open(
nugetFilePath,
ZipArchiveMode.Update
);
await RemoveContentFiles(nugetPackage);
await CleanDependencies(nugetPackage);
Console.WriteLine("Finishing package and copying to output directory");
nugetPackage.Dispose();
File.Move(nugetFilePath, Path.Combine(outputLocation, Path.GetFileName(nugetFilePath)), true);
}
Console.WriteLine("Building and packing frontend projects");
foreach (var frontendProject in frontendProjects)
{
await BuildProject(
frontendProject,
buildConfiguration
);
var nugetFilePath = await PackProject(
frontendProject,
TmpDir,
buildConfiguration
);
var nugetPackage = ZipFile.Open(
nugetFilePath,
ZipArchiveMode.Update
);
foreach (var entry in nugetPackage.Entries.ToArray())
{
if (!entry.FullName.StartsWith("staticwebassets/_framework"))
continue;
Console.WriteLine($"Removing framework file: {entry.FullName}");
entry.Delete();
}
var buildTargetEntry = nugetPackage.Entries.FirstOrDefault(x =>
x.FullName == "build/Microsoft.AspNetCore.StaticWebAssets.props"
);
if (buildTargetEntry != null)
{
Console.WriteLine("Removing framework file references");
await ModifyXmlInPackage(nugetPackage, buildTargetEntry,
document => document
.Descendants("StaticWebAsset")
.Where(x =>
{
var relativePath = x.Element("RelativePath")!.Value;
if (relativePath.StartsWith("_framework"))
return true;
if (relativePath.StartsWith("css/style.min.css"))
return true;
return false;
})
);
}
await CleanDependencies(nugetPackage);
await RemoveContentFiles(nugetPackage);
// Pack razor and html files into src folder
var additionalSrcFiles = new List<string>();
var basePath = Path.GetDirectoryName(frontendProject)!;
additionalSrcFiles.AddRange(
Directory.GetFiles(basePath, "*.razor", SearchOption.AllDirectories)
);
additionalSrcFiles.AddRange(
Directory.GetFiles(basePath, "index.html", SearchOption.AllDirectories)
);
foreach (var additionalSrcFile in additionalSrcFiles)
{
var relativePath = "src/" + additionalSrcFile.Replace(basePath, "").Trim('/');
Console.WriteLine($"Adding additional files as src: {relativePath}");
await using var fs = File.Open(
additionalSrcFile,
FileMode.Open,
FileAccess.ReadWrite,
FileShare.ReadWrite
);
var entry = nugetPackage.CreateEntry(relativePath);
await using var entryFs = entry.Open();
await fs.CopyToAsync(entryFs);
await entryFs.FlushAsync();
fs.Close();
entryFs.Close();
}
Console.WriteLine("Finishing package and copying to output directory");
nugetPackage.Dispose();
File.Move(nugetFilePath, Path.Combine(outputLocation, Path.GetFileName(nugetFilePath)), true);
}
Console.WriteLine("Building and packing shared projects");
foreach (var sharedProject in sharedProjects)
{
await BuildProject(
sharedProject,
buildConfiguration
);
var nugetFilePath = await PackProject(
sharedProject,
TmpDir,
buildConfiguration
);
File.Move(nugetFilePath, Path.Combine(outputLocation, Path.GetFileName(nugetFilePath)), true);
}
}
private async Task BuildProject(string file, string configuration)
{
var basePath = Path.GetFullPath(Path.GetDirectoryName(file)!);
var fileName = Path.GetFileName(file);
await CommandHelper.Run(
"/usr/bin/dotnet",
$"build {fileName} --configuration {configuration}",
basePath
);
}
private async Task<string> PackProject(string file, string output, string configuration)
{
var basePath = Path.GetFullPath(Path.GetDirectoryName(file)!);
var fileName = Path.GetFileName(file);
await CommandHelper.Run(
"/usr/bin/dotnet",
$"pack {fileName} --output {output} --configuration {configuration}",
basePath
);
var nugetFilesPaths = Directory.GetFiles(TmpDir, "*.nupkg", SearchOption.TopDirectoryOnly);
if (nugetFilesPaths.Length == 0)
throw new Exception("No nuget packages were built");
if (nugetFilesPaths.Length > 1)
throw new Exception("More than one nuget package has been built");
return nugetFilesPaths.First();
}
private async Task CleanDependencies(ZipArchive nugetPackage)
{
var nuspecEntry = nugetPackage.Entries.FirstOrDefault(x => x.Name.EndsWith(".nuspec"));
if (nuspecEntry == null)
{
Console.WriteLine("No nuspec file to modify found in nuget package");
return;
}
await ModifyXmlInPackage(nugetPackage, nuspecEntry, document =>
{
var ns = document.Root!.GetDefaultNamespace();
var metadata = document.Root!.Element(ns + "metadata")!;
var id = metadata.Element(ns + "id")!.Value;
// Skip the removal of moonlight references when
// we are packing moonlight itself
if (id.StartsWith("Moonlight."))
return [];
return document
.Descendants(ns + "dependency")
.Where(x => x.Attribute("id")?.Value.StartsWith("Moonlight.") ?? false);
});
}
private async Task<ZipArchiveEntry> ModifyXmlInPackage(
ZipArchive archive,
ZipArchiveEntry entry,
Func<XDocument, IEnumerable<XElement>> filter
)
{
var oldPath = entry.FullName;
await using var oldFs = entry.Open();
var document = await XDocument.LoadAsync(
oldFs,
LoadOptions.None,
CancellationToken.None
);
var itemsToRemove = filter.Invoke(document);
var items = itemsToRemove.ToArray();
foreach (var item in items)
item.Remove();
oldFs.Close();
entry.Delete();
var newEntry = archive.CreateEntry(oldPath);
var newFs = newEntry.Open();
await document.SaveAsync(newFs, SaveOptions.None, CancellationToken.None);
await newFs.FlushAsync();
newFs.Close();
return newEntry;
}
private async Task RemoveContentFiles(ZipArchive nugetPackage)
{
// Remove all content files
foreach (var entry in nugetPackage.Entries.ToArray())
{
if (!entry.FullName.StartsWith("contentFiles") && !entry.FullName.StartsWith("content"))
continue;
Console.WriteLine($"Removing content file: {entry.FullName}");
entry.Delete();
}
// Remove references to those files in the nuspec file
var nuspecFile = nugetPackage.Entries.FirstOrDefault(x => x.Name.EndsWith(".nuspec"));
if (nuspecFile != null)
{
Console.WriteLine("Removing references to content files in the nuspec files");
await ModifyXmlInPackage(
nugetPackage,
nuspecFile,
document =>
{
var ns = document.Root!.GetDefaultNamespace();
return document.Descendants(ns + "contentFiles");
}
);
}
}
}

View File

@@ -0,0 +1,414 @@
using System.IO.Compression;
using System.Text;
using System.Xml.Linq;
using Cocona;
using Microsoft.CodeAnalysis;
using Microsoft.CodeAnalysis.CSharp;
using Microsoft.CodeAnalysis.CSharp.Syntax;
using Scripts.Helpers;
namespace Scripts.Commands;
public class PreBuildCommand
{
private readonly CommandHelper CommandHelper;
private const string GeneratedStart = "// MLBUILD Generated Start";
private const string GeneratedEnd = "// MLBUILD Generated End";
private const string GeneratedHook = "// MLBUILD_PLUGIN_STARTUP_HERE";
public PreBuildCommand(CommandHelper commandHelper)
{
CommandHelper = commandHelper;
}
[Command("prebuild")]
public async Task Prebuild(
[Argument] string moonlightDir,
[Argument] string nugetDir
)
{
var dependencies = await GetDependenciesFromNuget(nugetDir);
Console.WriteLine("Following plugins found:");
foreach (var dependency in dependencies)
{
Console.WriteLine($"{dependency.Id} ({dependency.Version}) [{dependency.Tags}]");
}
var csProjFiles = Directory.GetFiles(moonlightDir, "*.csproj", SearchOption.AllDirectories);
try
{
Console.WriteLine("Adjusting csproj files");
foreach (var csProjFile in csProjFiles)
{
await using var fs = File.Open(
csProjFile,
FileMode.Open,
FileAccess.ReadWrite,
FileShare.ReadWrite
);
var document = await XDocument.LoadAsync(fs, LoadOptions.None, CancellationToken.None);
fs.Close();
// Search for tag definitions
var packageTagsElements = document.Descendants("PackageTags").ToArray();
if (packageTagsElements.Length == 0)
{
Console.WriteLine($"No package tags found in {Path.GetFullPath(csProjFile)}. Skipping it");
continue;
}
var packageTags = packageTagsElements.First().Value;
var dependenciesToAdd = dependencies
.Where(x => x.Tags.Contains(packageTags, StringComparison.InvariantCultureIgnoreCase))
.ToArray();
await RemoveDependencies(csProjFile);
await AddDependencies(csProjFile, dependenciesToAdd);
}
Console.WriteLine("Restoring projects");
foreach (var csProjFile in csProjFiles)
{
await RestoreProject(csProjFile, nugetDir);
}
Console.WriteLine("Generating plugin startup");
string[] validTags = ["apiserver", "frontend"];
foreach (var currentTag in validTags)
{
Console.WriteLine($"Checking for '{currentTag}' projects");
foreach (var csProjFile in csProjFiles)
{
var tags = await GetTagsFromCsproj(csProjFile);
if (string.IsNullOrEmpty(tags))
{
Console.WriteLine($"No package tags found in {Path.GetFullPath(csProjFile)}. Skipping it");
continue;
}
if (!tags.Contains(currentTag))
continue;
var currentDeps = dependencies
.Where(x => x.Tags.Contains(currentTag))
.ToArray();
var classPaths = await FindStartupClasses(currentDeps);
var code = new StringBuilder();
code.AppendLine(GeneratedStart);
foreach (var path in classPaths)
code.AppendLine($"pluginStartups.Add(new global::{path}());");
code.AppendLine(GeneratedEnd);
var filesToSearch = Directory.GetFiles(
Path.GetDirectoryName(csProjFile)!,
"*.cs",
SearchOption.AllDirectories
);
foreach (var file in filesToSearch)
{
var content = await File.ReadAllTextAsync(file);
if (!content.Contains(GeneratedHook, StringComparison.InvariantCultureIgnoreCase))
continue;
Console.WriteLine($"Injecting generated code to: {Path.GetFullPath(file)}");
content = content.Replace(
GeneratedHook,
code.ToString(),
StringComparison.InvariantCultureIgnoreCase
);
await File.WriteAllTextAsync(file, content);
}
}
}
}
catch (Exception)
{
Console.WriteLine("An error occured while prebuilding moonlight. Removing csproj modifications");
foreach (var csProjFile in csProjFiles)
await RemoveDependencies(csProjFile);
await RemoveGeneratedCode(moonlightDir);
throw;
}
}
[Command("prebuild-reset")]
public async Task PrebuildReset(
[Argument] string moonlightDir
)
{
var csProjFiles = Directory.GetFiles(moonlightDir, "*.csproj", SearchOption.AllDirectories);
Console.WriteLine("Reverting csproj changes");
foreach (var csProjFile in csProjFiles)
await RemoveDependencies(csProjFile);
Console.WriteLine("Removing generated code");
await RemoveGeneratedCode(moonlightDir);
}
[Command("test")]
public async Task Test(
[Argument] string nugetDir
)
{
var dependencies = await GetDependenciesFromNuget(nugetDir);
await FindStartupClasses(dependencies);
}
private async Task<Dependency[]> GetDependenciesFromNuget(string nugetDir)
{
var nugetFiles = Directory.GetFiles(nugetDir, "*.nupkg", SearchOption.AllDirectories);
var dependencies = new List<Dependency>();
foreach (var nugetFile in nugetFiles)
{
var dependency = await GetDependencyFromPackage(nugetFile);
dependencies.Add(dependency);
}
return dependencies.ToArray();
}
private async Task<Dependency> GetDependencyFromPackage(string path)
{
using var nugetPackage = ZipFile.Open(path, ZipArchiveMode.Read);
var nuspecEntry = nugetPackage.Entries.First(x => x.Name.EndsWith(".nuspec"));
await using var nuspecFs = nuspecEntry.Open();
var nuspec = await XDocument.LoadAsync(nuspecFs, LoadOptions.None, CancellationToken.None);
var ns = nuspec.Root!.GetDefaultNamespace();
var metadata = nuspec.Root!.Element(ns + "metadata")!;
var id = metadata.Element(ns + "id")!.Value;
var version = metadata.Element(ns + "version")!.Value;
var tags = metadata.Element(ns + "tags")!.Value;
nuspecFs.Close();
return new Dependency()
{
Id = id,
Version = version,
Tags = tags
};
}
private async Task AddDependencies(string path, Dependency[] dependencies)
{
await using var fs = File.Open(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.ReadWrite);
fs.Position = 0;
var csProj = await XDocument.LoadAsync(fs, LoadOptions.None, CancellationToken.None);
var project = csProj.Element("Project")!;
var itemGroup = new XElement("ItemGroup");
itemGroup.SetAttributeValue("Label", "MoonlightBuildDeps");
foreach (var dependency in dependencies)
{
var depElement = new XElement("PackageReference");
depElement.SetAttributeValue("Include", dependency.Id);
depElement.SetAttributeValue("Version", dependency.Version);
itemGroup.Add(depElement);
}
project.Add(itemGroup);
fs.Position = 0;
await csProj.SaveAsync(fs, SaveOptions.None, CancellationToken.None);
}
private Task RemoveDependencies(string path)
{
var csProj = XDocument.Load(path, LoadOptions.None);
var itemGroupsToRemove = csProj
.Descendants("ItemGroup")
.Where(x => x.Attribute("Label")?.Value.Contains("MoonlightBuildDeps") ?? false)
.ToArray();
itemGroupsToRemove.Remove();
csProj.Save(path, SaveOptions.None);
return Task.CompletedTask;
}
private async Task RestoreProject(string file, string nugetPath)
{
var basePath = Path.GetFullPath(Path.GetDirectoryName(file)!);
var fileName = Path.GetFileName(file);
var nugetPathFull = Path.GetFullPath(nugetPath);
Console.WriteLine($"Restore: {basePath} - {fileName}");
await CommandHelper.Run(
"/usr/bin/dotnet",
$"restore {fileName} --source {nugetPathFull}",
basePath
);
}
private async Task<string[]> FindStartupClasses(Dependency[] dependencies)
{
var result = new List<string>();
var nugetPath = Path.Combine(
Environment.GetFolderPath(Environment.SpecialFolder.UserProfile),
".nuget",
"packages"
);
var filesToScan = dependencies.SelectMany(dependency =>
{
var dependencySrcPath = Path.Combine(nugetPath, dependency.Id.ToLower(), dependency.Version, "src");
Console.WriteLine($"Checking {dependencySrcPath}");
if (!Directory.Exists(dependencySrcPath))
return [];
return Directory.GetFiles(dependencySrcPath, "*.cs", SearchOption.AllDirectories);
}
).ToArray();
var mscorlib = MetadataReference.CreateFromFile(typeof(object).Assembly.Location);
var trees = new List<SyntaxTree>();
foreach (var file in filesToScan)
{
Console.WriteLine($"Reading {file}");
var content = await File.ReadAllTextAsync(file);
var tree = CSharpSyntaxTree.ParseText(content);
trees.Add(tree);
}
var compilation = CSharpCompilation.Create("Analysis", trees, [mscorlib]);
foreach (var tree in trees)
{
var model = compilation.GetSemanticModel(tree);
var root = await tree.GetRootAsync();
var classDeclarations = root
.DescendantNodes()
.OfType<ClassDeclarationSyntax>();
foreach (var classDeclaration in classDeclarations)
{
var symbol = model.GetDeclaredSymbol(classDeclaration);
if (symbol == null)
continue;
var hasAttribute = symbol.GetAttributes().Any(attr =>
{
if (attr.AttributeClass == null)
return false;
return attr.AttributeClass.Name == "PluginStartup";
});
if (!hasAttribute)
continue;
var classPath =
$"{symbol.ContainingNamespace.ToDisplayString()}.{classDeclaration.Identifier.ValueText}";
Console.WriteLine($"Detected startup in class: {classPath}");
result.Add(classPath);
}
}
return result.ToArray();
}
private async Task<string> GetTagsFromCsproj(string csProjFile)
{
await using var fs = File.Open(
csProjFile,
FileMode.Open,
FileAccess.ReadWrite,
FileShare.ReadWrite
);
var document = await XDocument.LoadAsync(fs, LoadOptions.None, CancellationToken.None);
fs.Close();
// Search for tag definitions
var packageTagsElements = document.Descendants("PackageTags").ToArray();
if (packageTagsElements.Length == 0)
return "";
return packageTagsElements.First().Value;
}
private async Task RemoveGeneratedCode(string dir)
{
var filesToSearch = Directory.GetFiles(
dir,
"*.cs",
SearchOption.AllDirectories
);
foreach (var file in filesToSearch)
{
// We dont want to replace ourself
if (file.Contains("PreBuildCommand.cs"))
continue;
var content = await File.ReadAllTextAsync(file);
if (!content.Contains(GeneratedStart) || !content.Contains(GeneratedEnd))
continue;
var startIndex = content.IndexOf(GeneratedStart, StringComparison.InvariantCultureIgnoreCase);
var endIndex = content.IndexOf(GeneratedEnd, startIndex, StringComparison.InvariantCultureIgnoreCase) +
GeneratedEnd.Length;
var cutOut = content.Substring(startIndex, endIndex - startIndex);
content = content.Replace(cutOut, GeneratedHook);
await File.WriteAllTextAsync(file, content);
}
}
private record Dependency
{
public string Id { get; set; }
public string Version { get; set; }
public string Tags { get; set; }
}
}