using System; using System.IO; using System.Text; using System.Text.Json; using System.Threading; namespace Editor; public interface IProgress { public void SetProgressMessage( string message ) { } public void SetProgress( float total, float current ) { } } public partial class ProjectPublisher { public class PackageManifest { public string Summary { get; set; } public string Description { get; set; } public bool IncludeSourceFiles { get; set; } /// /// List of packages that the code references /// public HashSet CodePackageReferences { get; } = new(); public List Errors = new List(); IProgress progress; ulong scannedBytes; public ProjectFile FindAsset( string relativePath ) { return Assets.FirstOrDefault( x => string.Equals( x.Name, relativePath, StringComparison.OrdinalIgnoreCase ) ); } public List Assets { get; set; } = new(); public async Task BuildFromAssets( Project project, IProgress progress = null, CancellationToken cancel = default ) { Assets.Clear(); var rootFolder = project.RootDirectory.FullName; this.progress = progress; if ( !string.IsNullOrWhiteSpace( project.Config.Resources ) ) { cancel.ThrowIfCancellationRequested(); foreach ( var path in AllAssetPaths( project ) ) { await IncludeFiles( path, project.Config.Resources, cancel ); } } // // Collect localization files // await IncludeFiles( rootFolder, "Localization/*.json", cancel ); await Task.Delay( 10 ); cancel.ThrowIfCancellationRequested(); // // Collect font files // foreach ( var path in AllAssetPaths( project ) ) { await IncludeFiles( path, "fonts/*", cancel ); } await Task.Delay( 10 ); cancel.ThrowIfCancellationRequested(); // // Include all project config // if ( project.Config.Type == "game" ) { await IncludeFiles( rootFolder, "ProjectSettings/*", cancel ); } await Task.Delay( 10 ); cancel.ThrowIfCancellationRequested(); // // If we're a game, include content from /addons/base/code/ // This versions things like the styles and dev ui, which is only // going to work with the shipped base code. // if ( project.Config.Type == "game" ) { progress?.SetProgressMessage( "Collecting base code assets" ); await IncludeFiles( FileSystem.Root.GetFullPath( "/addons/base/code/" ), "*", cancel ); } await Task.Delay( 10 ); cancel.ThrowIfCancellationRequested(); // // Search the code path for files // foreach ( var path in AllCodePaths( project ) ) { progress?.SetProgressMessage( "Collecting code assets" ); await IncludeFiles( path, "*", cancel ); } await Task.Delay( 10 ); cancel.ThrowIfCancellationRequested(); // // Search in assets // { progress?.SetProgressMessage( "Collecting assets" ); await CollectAssets( project, cancel ); } this.progress = null; } IEnumerable AllCodePaths( Project project ) { if ( project.HasCodePath() ) yield return project.GetCodePath(); // each library foreach ( var library in LibrarySystem.All ) { if ( library.Project.HasCodePath() ) yield return library.Project.GetCodePath(); } } IEnumerable AllAssetPaths( Project project ) { if ( project.HasAssetsPath() ) yield return project.GetAssetsPath(); // each library foreach ( var library in LibrarySystem.All ) { if ( library.Project.HasAssetsPath() ) yield return library.Project.GetAssetsPath(); } } internal async Task BuildFrom( Asset singleAsset, CancellationToken cancel = default ) { Assets.Clear(); var assetList = new List(); assetList.Add( singleAsset ); await CollectAssets( assetList, cancel ); progress = null; if ( Assets.Count == 0 ) Errors.Add( "No files found" ); } public async Task BuildFromSource( Project addon, IProgress progress = null, CancellationToken cancel = default ) { Assets.Clear(); var rootFolder = addon.RootDirectory.FullName; this.progress = progress; // // Collect localization files // await IncludeFiles( rootFolder, "*", cancel, true ); cancel.ThrowIfCancellationRequested(); this.progress = null; } public string ToJson() => JsonSerializer.Serialize( this, new JsonSerializerOptions( JsonSerializerOptions.Default ) { WriteIndented = true } ); private async Task CollectAssets( Project project, CancellationToken cancel ) { foreach ( var path in AllAssetPaths( project ) ) { var assetPath = path.Replace( '\\', '/' ); assetPath = assetPath.TrimEnd( '/' ) + '/'; progress?.SetProgressMessage( "Finding Assets.." ); var assets = AssetSystem.All.Where( x => x.AbsolutePath.StartsWith( assetPath, StringComparison.OrdinalIgnoreCase ) ).ToList(); await CollectAssets( assets, cancel ); } } /// /// Collect and add input dependencies to the manifest. /// These are files that were involved in compile but don't cause a recompile - usually only present for child resources (ie. the tga that a vtex_c came from) /// /// /// private async Task CollectInputDependencies( Asset asset ) { if ( !IncludeSourceFiles ) return; foreach ( var file in asset.GetAdditionalRelatedFiles() ) { if ( !IncludeSourceFiles && !file.EndsWith( ".rect" ) ) continue; var ast = AssetSystem.FindByPath( file ); if ( ast == null ) continue; await AddFile( ast.AbsolutePath, ast.RelativePath ); } foreach ( var a in asset.GetInputDependencies() ) { var ast = AssetSystem.FindByPath( a ); if ( ast == null ) continue; await AddFile( ast.AbsolutePath, ast.RelativePath ); } } private async Task CollectAssets( List assets, CancellationToken cancel ) { HashSet AddedAssets = new(); foreach ( var asset in assets ) { cancel.ThrowIfCancellationRequested(); AddedAssets.Add( asset ); foreach ( var a in asset.GetReferences( true ) ) { AddedAssets.Add( a ); await CollectInputDependencies( a ); } foreach ( var file in asset.GetAdditionalRelatedFiles() ) { if ( !IncludeSourceFiles && !file.EndsWith( ".rect" ) ) continue; var ast = AssetSystem.FindByPath( file ); if ( ast == null ) continue; await CollectInputDependencies( ast ); await AddFile( ast.AbsolutePath, ast.RelativePath ); } progress?.SetProgressMessage( $"Found {AddedAssets.Count:n0}" ); } List tasks = new List(); int i = 0; foreach ( var a in AddedAssets ) { cancel.ThrowIfCancellationRequested(); progress?.SetProgress( i++, AddedAssets.Count ); progress?.SetProgressMessage( $"Adding Asset {i:n0} of {AddedAssets.Count:n0} - {a.RelativePath}" ); tasks.Add( AddAsset( a ) ); while ( tasks.Count > 8 ) { await Task.WhenAny( tasks.ToArray() ); tasks.RemoveAll( x => x.IsCompleted ); } } await Task.WhenAll( tasks.ToArray() ); } async Task AddAsset( Asset asset ) { if ( !CanPublishFile( asset ) ) return false; await asset.CompileIfNeededAsync(); if ( IncludeSourceFiles ) { var abs = asset.GetSourceFile( true ); var rel = asset.GetSourceFile( false ); await AddFile( abs, rel ); } { var abs = asset.GetCompiledFile( true ); var rel = asset.GetCompiledFile( false ); if ( asset.IsCompileFailed ) { Errors.Add( $"Asset failed to compile: {asset.Path}" ); return false; } // // This is fine, some shit doesn't get compiled. // There's probably a way to find this out proper though. // if ( string.IsNullOrEmpty( abs ) ) { //Log.Warning( $"Compiled file missing: {asset.Path}" ); return false; } // Add this file await AddFile( abs, rel ); } return true; } private async Task AddFile( string absPath, string relativePath ) { if ( !System.IO.File.Exists( absPath ) ) { Errors.Add( $"File not found \"{absPath}\" ({relativePath})" ); return; } relativePath = relativePath.NormalizeFilename( false, false ).TrimStart( '/' ); // // already added // if ( Assets.Any( x => string.Equals( x.Name, relativePath, StringComparison.OrdinalIgnoreCase ) ) ) return; var info = new System.IO.FileInfo( absPath ); var e = new ProjectFile { Name = relativePath, Size = (int)info.Length, AbsolutePath = absPath }; // run in a thread to make it super fast await Task.Run( async () => { using ( var stream = info.OpenRead() ) { e.Hash = (await Sandbox.Utility.Crc64.FromStreamAsync( stream )).ToString( "x" ); } } ); scannedBytes += (ulong)e.Size; Assets.Add( e ); } internal async Task IncludeFiles( string root, string wildcardScript, CancellationToken cancel, bool allowSourceFiles = false ) { if ( !System.IO.Directory.Exists( root ) ) return; var wildcards = wildcardScript.Split( "\n", StringSplitOptions.RemoveEmptyEntries ) .Select( x => x.Trim() ) .Where( x => !x.StartsWith( "//" ) ) .Select( x => x.NormalizeFilename( true, false ) ) .ToArray(); foreach ( var file in System.IO.Directory.EnumerateFiles( root, "*", SearchOption.AllDirectories ) ) { var relative = System.IO.Path.GetRelativePath( root, file ).NormalizeFilename( true, false ); if ( !LooseFileAllowed( relative, allowSourceFiles ) ) continue; if ( !wildcards.Any( x => relative.WildcardMatch( x ) ) ) continue; if ( new System.IO.FileInfo( file ).Length < 1 ) continue; await AddFile( file, relative ); cancel.ThrowIfCancellationRequested(); } } /// /// This really exists only to dissallow dangerous extensions like .exe etc. /// So feel free to add anything non dangerous to this list. /// public static string[] DissallowedExtensions = new string[] { ".dll", ".exe", ".csproj", ".sln", ".user", ".slnx" }; public static bool LooseFileAllowed( string file, bool allowSourceFiles ) { if ( file.Contains( "/obj/", StringComparison.OrdinalIgnoreCase ) ) return false; if ( file.Contains( "/.git", StringComparison.OrdinalIgnoreCase ) ) return false; if ( file.Contains( "/.addon", StringComparison.OrdinalIgnoreCase ) ) return false; if ( file.Contains( "/.editorconfig", StringComparison.OrdinalIgnoreCase ) ) return false; if ( file.Contains( "/.vs/", StringComparison.OrdinalIgnoreCase ) ) return false; if ( file.Contains( "_bakeresourcecache", StringComparison.OrdinalIgnoreCase ) ) return false; if ( file.Contains( "launchsettings.json", StringComparison.OrdinalIgnoreCase ) ) return false; if ( !allowSourceFiles ) { if ( file.Contains( ".sbproj", StringComparison.OrdinalIgnoreCase ) ) return false; if ( file.EndsWith( ".cs", StringComparison.OrdinalIgnoreCase ) ) return false; if ( file.EndsWith( ".razor", StringComparison.OrdinalIgnoreCase ) ) return false; if ( file.EndsWith( ".fbx", StringComparison.OrdinalIgnoreCase ) ) return false; } if ( DissallowedExtensions.Any( x => file.EndsWith( x, StringComparison.OrdinalIgnoreCase ) ) ) return false; return true; } public async Task AddTextFile( string contents, string relativePath ) { var bytes = Encoding.UTF8.GetBytes( contents ); await AddFile( bytes, relativePath ); } internal async Task AddFile( byte[] contents, string relativePath ) { var e = new ProjectFile { Name = relativePath, Size = (int)contents.Length, Contents = contents }; // run in a thread to make it super fast await Task.Run( async () => { using ( var stream = new MemoryStream( contents ) ) { e.Hash = (await Sandbox.Utility.Crc64.FromStreamAsync( stream )).ToString( "x" ); } } ); scannedBytes += (ulong)e.Size; Assets.Add( e ); } /// /// Test our wildcards and make sure to pull in any assets that the assets that /// we're whitelisting are referencing. If they're not already included in the wildcard /// then we'll add them by full relative path to the end of the list. /// internal string[] GrabWildcardReferences( string wildcard ) { var script = wildcard ?? ""; var parts = script .Split( "\n", StringSplitOptions.RemoveEmptyEntries ) .Where( x => !x.StartsWith( "//" ) ) .ToHashSet( StringComparer.OrdinalIgnoreCase ); // get a list of included assets that match this wildcard system var assets = Assets.Where( x => parts.Any( y => x.Name.WildcardMatch( y ) ) ).ToArray(); // loop each hit asset and get references. foreach ( var a in assets ) { var asset = AssetSystem.FindByPath( a.AbsolutePath ); if ( asset == null ) continue; foreach ( var d in asset.GetReferences( true ) ) { // aleady have this reference if ( assets.Any( x => x.AbsolutePath == d.AbsolutePath ) ) continue; // add it to the end parts.Add( d.Path.Replace( "\\", "/" ).TrimStart( '/' ) ); } } // return collapsed version return parts.ToArray(); } /// /// We're referencing this asset package, so add it as an EditorReference and /// include its asset. /// internal async Task AddCodePackageReference( string package ) { var asset = await AssetSystem.InstallAsync( package ); if ( asset == null ) { Log.Warning( $"Couldn't find asset for package {package}" ); return; } CodePackageReferences.Add( package.ToLower() ); await AddAsset( asset ); foreach ( var a in asset.GetReferences( true ) ) { await AddAsset( a ); } } } }