using System;
using System.IO;
using System.Text;
using System.Text.Json;
using System.Threading;
namespace Editor;
public interface IProgress
{
public void SetProgressMessage( string message ) { }
public void SetProgress( float total, float current ) { }
}
public partial class ProjectPublisher
{
public class PackageManifest
{
public string Summary { get; set; }
public string Description { get; set; }
public bool IncludeSourceFiles { get; set; }
///
/// List of packages that the code references
///
public HashSet CodePackageReferences { get; } = new();
public List Errors = new List();
IProgress progress;
ulong scannedBytes;
public ProjectFile FindAsset( string relativePath )
{
return Assets.FirstOrDefault( x => string.Equals( x.Name, relativePath, StringComparison.OrdinalIgnoreCase ) );
}
public List Assets { get; set; } = new();
public async Task BuildFromAssets( Project project, IProgress progress = null, CancellationToken cancel = default )
{
Assets.Clear();
var rootFolder = project.RootDirectory.FullName;
this.progress = progress;
if ( !string.IsNullOrWhiteSpace( project.Config.Resources ) )
{
cancel.ThrowIfCancellationRequested();
foreach ( var path in AllAssetPaths( project ) )
{
await IncludeFiles( path, project.Config.Resources, cancel );
}
}
//
// Collect localization files
//
await IncludeFiles( rootFolder, "Localization/*.json", cancel );
await Task.Delay( 10 );
cancel.ThrowIfCancellationRequested();
//
// Collect font files
//
foreach ( var path in AllAssetPaths( project ) )
{
await IncludeFiles( path, "fonts/*", cancel );
}
await Task.Delay( 10 );
cancel.ThrowIfCancellationRequested();
//
// Include all project config
//
if ( project.Config.Type == "game" )
{
await IncludeFiles( rootFolder, "ProjectSettings/*", cancel );
}
await Task.Delay( 10 );
cancel.ThrowIfCancellationRequested();
//
// If we're a game, include content from /addons/base/code/
// This versions things like the styles and dev ui, which is only
// going to work with the shipped base code.
//
if ( project.Config.Type == "game" )
{
progress?.SetProgressMessage( "Collecting base code assets" );
await IncludeFiles( FileSystem.Root.GetFullPath( "/addons/base/code/" ), "*", cancel );
}
await Task.Delay( 10 );
cancel.ThrowIfCancellationRequested();
//
// Search the code path for files
//
foreach ( var path in AllCodePaths( project ) )
{
progress?.SetProgressMessage( "Collecting code assets" );
await IncludeFiles( path, "*", cancel );
}
await Task.Delay( 10 );
cancel.ThrowIfCancellationRequested();
//
// Search in assets
//
{
progress?.SetProgressMessage( "Collecting assets" );
await CollectAssets( project, cancel );
}
this.progress = null;
}
IEnumerable AllCodePaths( Project project )
{
if ( project.HasCodePath() )
yield return project.GetCodePath();
// each library
foreach ( var library in LibrarySystem.All )
{
if ( library.Project.HasCodePath() )
yield return library.Project.GetCodePath();
}
}
IEnumerable AllAssetPaths( Project project )
{
if ( project.HasAssetsPath() )
yield return project.GetAssetsPath();
// each library
foreach ( var library in LibrarySystem.All )
{
if ( library.Project.HasAssetsPath() )
yield return library.Project.GetAssetsPath();
}
}
internal async Task BuildFrom( Asset singleAsset, CancellationToken cancel = default )
{
Assets.Clear();
var assetList = new List();
assetList.Add( singleAsset );
await CollectAssets( assetList, cancel );
progress = null;
if ( Assets.Count == 0 )
Errors.Add( "No files found" );
}
public async Task BuildFromSource( Project addon, IProgress progress = null, CancellationToken cancel = default )
{
Assets.Clear();
var rootFolder = addon.RootDirectory.FullName;
this.progress = progress;
//
// Collect localization files
//
await IncludeFiles( rootFolder, "*", cancel, true );
cancel.ThrowIfCancellationRequested();
this.progress = null;
}
public string ToJson() => JsonSerializer.Serialize( this, new JsonSerializerOptions( JsonSerializerOptions.Default ) { WriteIndented = true } );
private async Task CollectAssets( Project project, CancellationToken cancel )
{
foreach ( var path in AllAssetPaths( project ) )
{
var assetPath = path.Replace( '\\', '/' );
assetPath = assetPath.TrimEnd( '/' ) + '/';
progress?.SetProgressMessage( "Finding Assets.." );
var assets = AssetSystem.All.Where( x => x.AbsolutePath.StartsWith( assetPath, StringComparison.OrdinalIgnoreCase ) ).ToList();
await CollectAssets( assets, cancel );
}
}
///
/// Collect and add input dependencies to the manifest.
/// These are files that were involved in compile but don't cause a recompile - usually only present for child resources (ie. the tga that a vtex_c came from)
///
///
///
private async Task CollectInputDependencies( Asset asset )
{
if ( !IncludeSourceFiles ) return;
foreach ( var file in asset.GetAdditionalRelatedFiles() )
{
if ( !IncludeSourceFiles && !file.EndsWith( ".rect" ) )
continue;
var ast = AssetSystem.FindByPath( file );
if ( ast == null ) continue;
await AddFile( ast.AbsolutePath, ast.RelativePath );
}
foreach ( var a in asset.GetInputDependencies() )
{
var ast = AssetSystem.FindByPath( a );
if ( ast == null ) continue;
await AddFile( ast.AbsolutePath, ast.RelativePath );
}
}
private async Task CollectAssets( List assets, CancellationToken cancel )
{
HashSet AddedAssets = new();
foreach ( var asset in assets )
{
cancel.ThrowIfCancellationRequested();
AddedAssets.Add( asset );
foreach ( var a in asset.GetReferences( true ) )
{
AddedAssets.Add( a );
await CollectInputDependencies( a );
}
foreach ( var file in asset.GetAdditionalRelatedFiles() )
{
if ( !IncludeSourceFiles && !file.EndsWith( ".rect" ) )
continue;
var ast = AssetSystem.FindByPath( file );
if ( ast == null ) continue;
await CollectInputDependencies( ast );
await AddFile( ast.AbsolutePath, ast.RelativePath );
}
progress?.SetProgressMessage( $"Found {AddedAssets.Count:n0}" );
}
List tasks = new List();
int i = 0;
foreach ( var a in AddedAssets )
{
cancel.ThrowIfCancellationRequested();
progress?.SetProgress( i++, AddedAssets.Count );
progress?.SetProgressMessage( $"Adding Asset {i:n0} of {AddedAssets.Count:n0} - {a.RelativePath}" );
tasks.Add( AddAsset( a ) );
while ( tasks.Count > 8 )
{
await Task.WhenAny( tasks.ToArray() );
tasks.RemoveAll( x => x.IsCompleted );
}
}
await Task.WhenAll( tasks.ToArray() );
}
async Task AddAsset( Asset asset )
{
if ( !CanPublishFile( asset ) )
return false;
await asset.CompileIfNeededAsync();
if ( IncludeSourceFiles )
{
var abs = asset.GetSourceFile( true );
var rel = asset.GetSourceFile( false );
await AddFile( abs, rel );
}
{
var abs = asset.GetCompiledFile( true );
var rel = asset.GetCompiledFile( false );
if ( asset.IsCompileFailed )
{
Errors.Add( $"Asset failed to compile: {asset.Path}" );
return false;
}
//
// This is fine, some shit doesn't get compiled.
// There's probably a way to find this out proper though.
//
if ( string.IsNullOrEmpty( abs ) )
{
//Log.Warning( $"Compiled file missing: {asset.Path}" );
return false;
}
// Add this file
await AddFile( abs, rel );
}
return true;
}
private async Task AddFile( string absPath, string relativePath )
{
if ( !System.IO.File.Exists( absPath ) )
{
Errors.Add( $"File not found \"{absPath}\" ({relativePath})" );
return;
}
relativePath = relativePath.NormalizeFilename( false, false ).TrimStart( '/' );
//
// already added
//
if ( Assets.Any( x => string.Equals( x.Name, relativePath, StringComparison.OrdinalIgnoreCase ) ) )
return;
var info = new System.IO.FileInfo( absPath );
var e = new ProjectFile
{
Name = relativePath,
Size = (int)info.Length,
AbsolutePath = absPath
};
// run in a thread to make it super fast
await Task.Run( async () =>
{
using ( var stream = info.OpenRead() )
{
e.Hash = (await Sandbox.Utility.Crc64.FromStreamAsync( stream )).ToString( "x" );
}
} );
scannedBytes += (ulong)e.Size;
Assets.Add( e );
}
internal async Task IncludeFiles( string root, string wildcardScript, CancellationToken cancel, bool allowSourceFiles = false )
{
if ( !System.IO.Directory.Exists( root ) )
return;
var wildcards = wildcardScript.Split( "\n", StringSplitOptions.RemoveEmptyEntries )
.Select( x => x.Trim() )
.Where( x => !x.StartsWith( "//" ) )
.Select( x => x.NormalizeFilename( true, false ) )
.ToArray();
foreach ( var file in System.IO.Directory.EnumerateFiles( root, "*", SearchOption.AllDirectories ) )
{
var relative = System.IO.Path.GetRelativePath( root, file ).NormalizeFilename( true, false );
if ( !LooseFileAllowed( relative, allowSourceFiles ) )
continue;
if ( !wildcards.Any( x => relative.WildcardMatch( x ) ) )
continue;
if ( new System.IO.FileInfo( file ).Length < 1 )
continue;
await AddFile( file, relative );
cancel.ThrowIfCancellationRequested();
}
}
///
/// This really exists only to dissallow dangerous extensions like .exe etc.
/// So feel free to add anything non dangerous to this list.
///
public static string[] DissallowedExtensions = new string[]
{
".dll", ".exe", ".csproj", ".sln", ".user", ".slnx"
};
public static bool LooseFileAllowed( string file, bool allowSourceFiles )
{
if ( file.Contains( "/obj/", StringComparison.OrdinalIgnoreCase ) ) return false;
if ( file.Contains( "/.git", StringComparison.OrdinalIgnoreCase ) ) return false;
if ( file.Contains( "/.addon", StringComparison.OrdinalIgnoreCase ) ) return false;
if ( file.Contains( "/.editorconfig", StringComparison.OrdinalIgnoreCase ) ) return false;
if ( file.Contains( "/.vs/", StringComparison.OrdinalIgnoreCase ) ) return false;
if ( file.Contains( "_bakeresourcecache", StringComparison.OrdinalIgnoreCase ) ) return false;
if ( file.Contains( "launchsettings.json", StringComparison.OrdinalIgnoreCase ) ) return false;
if ( !allowSourceFiles )
{
if ( file.Contains( ".sbproj", StringComparison.OrdinalIgnoreCase ) ) return false;
if ( file.EndsWith( ".cs", StringComparison.OrdinalIgnoreCase ) ) return false;
if ( file.EndsWith( ".razor", StringComparison.OrdinalIgnoreCase ) ) return false;
if ( file.EndsWith( ".fbx", StringComparison.OrdinalIgnoreCase ) ) return false;
}
if ( DissallowedExtensions.Any( x => file.EndsWith( x, StringComparison.OrdinalIgnoreCase ) ) )
return false;
return true;
}
public async Task AddTextFile( string contents, string relativePath )
{
var bytes = Encoding.UTF8.GetBytes( contents );
await AddFile( bytes, relativePath );
}
internal async Task AddFile( byte[] contents, string relativePath )
{
var e = new ProjectFile
{
Name = relativePath,
Size = (int)contents.Length,
Contents = contents
};
// run in a thread to make it super fast
await Task.Run( async () =>
{
using ( var stream = new MemoryStream( contents ) )
{
e.Hash = (await Sandbox.Utility.Crc64.FromStreamAsync( stream )).ToString( "x" );
}
} );
scannedBytes += (ulong)e.Size;
Assets.Add( e );
}
///
/// Test our wildcards and make sure to pull in any assets that the assets that
/// we're whitelisting are referencing. If they're not already included in the wildcard
/// then we'll add them by full relative path to the end of the list.
///
internal string[] GrabWildcardReferences( string wildcard )
{
var script = wildcard ?? "";
var parts = script
.Split( "\n", StringSplitOptions.RemoveEmptyEntries )
.Where( x => !x.StartsWith( "//" ) )
.ToHashSet( StringComparer.OrdinalIgnoreCase );
// get a list of included assets that match this wildcard system
var assets = Assets.Where( x => parts.Any( y => x.Name.WildcardMatch( y ) ) ).ToArray();
// loop each hit asset and get references.
foreach ( var a in assets )
{
var asset = AssetSystem.FindByPath( a.AbsolutePath );
if ( asset == null ) continue;
foreach ( var d in asset.GetReferences( true ) )
{
// aleady have this reference
if ( assets.Any( x => x.AbsolutePath == d.AbsolutePath ) )
continue;
// add it to the end
parts.Add( d.Path.Replace( "\\", "/" ).TrimStart( '/' ) );
}
}
// return collapsed version
return parts.ToArray();
}
///
/// We're referencing this asset package, so add it as an EditorReference and
/// include its asset.
///
internal async Task AddCodePackageReference( string package )
{
var asset = await AssetSystem.InstallAsync( package );
if ( asset == null )
{
Log.Warning( $"Couldn't find asset for package {package}" );
return;
}
CodePackageReferences.Add( package.ToLower() );
await AddAsset( asset );
foreach ( var a in asset.GetReferences( true ) )
{
await AddAsset( a );
}
}
}
}