sbox-public/engine/Tools/SboxBuild/Steps/SyncPublicRepo.cs

using Microsoft.Extensions.FileSystemGlobbing;
using System;
using System.Collections.Concurrent;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using static Facepunch.Constants;

namespace Facepunch.Steps;

internal record ArtifactFileInfo
{
	[JsonPropertyName( "path" )]
	public string Path { get; init; }

	[JsonPropertyName( "sha256" )]
	public string Sha256 { get; init; }

	[JsonPropertyName( "size" )]
	public long Size { get; init; }
}

internal record ArtifactManifest
{
	[JsonPropertyName( "commit" )]
	public string Commit { get; init; }

	[JsonPropertyName( "timestamp" )]
	public string Timestamp { get; init; }

	[JsonPropertyName( "files" )]
	public List<ArtifactFileInfo> Files { get; init; }
}

/// <summary>
/// Syncs the master branch to the public repository by filtering specific paths
/// </summary>
internal class SyncPublicRepo( string name, bool dryRun = false ) : Step( name )
{
	private const string PUBLIC_REPO = "Facepunch/sbox-public";
	private const string PUBLIC_BRANCH = "master";
	private const string SHALLOW_EXCLUDE_TAG = "public-history-root";
	private const int MAX_PARALLEL_UPLOADS = 32;

	protected override ExitCode RunInternal()
	{
		try
		{
			var success = SyncToPublicRepository();
			if ( !success )
			{
				return ExitCode.Failure;
			}

			return ExitCode.Success;
		}
		catch ( Exception ex )
		{
			Log.Error( $"Public repo sync failed with error: {ex}" );
			return ExitCode.Failure;
		}
	}

	private static readonly string[] RepoFilterPathIncludeGlobs =
	{
		"engine/**",
		"game/**",
		".editorconfig",
		"public/**"
	};

	private static readonly string[] RepoFilterPathExcludeGlobs =
	{
		"**/*.pdb",
		"game/core/shaders/**"
	};

	private static readonly string[] RepoFilterShaderWhitelistGlobs =
	{
		"game/core/shaders/**/vr_*",
		"game/core/shaders/**/*.shader_c",
		"game/core/shaders/common.fxc",
		"game/core/shaders/common_samplers.fxc",
		"game/core/shaders/descriptor_set_support.fxc",
		"game/core/shaders/system.fxc",
		"game/core/shaders/tiled_culling.hlsl"
	};

	private static readonly Dictionary<string, string> RepoFilterPathRenames = new()
	{
		{ "public/.gitignore", ".gitignore" },
		{ "public/.gitattributes", ".gitattributes" },
		{ "public/README.md", "README.md" },
		{ "public/LICENSE.md", "LICENSE.md" },
		{ "public/CONTRIBUTING.md", "CONTRIBUTING.md" },
		{ "public/Bootstrap.bat", "Bootstrap.bat" }
	};

	private static Matcher RepoFileFilter()
	{
		if ( _matcher is not null )
		{
			return _matcher;
		}

		// Ordered since we first include everything, then exclude, then re-include specific files
		_matcher = new Matcher( StringComparison.OrdinalIgnoreCase, preserveFilterOrder: true );

		_matcher.AddIncludePatterns( RepoFilterPathIncludeGlobs );

		_matcher.AddExcludePatterns( RepoFilterPathExcludeGlobs );

		_matcher.AddIncludePatterns( RepoFilterShaderWhitelistGlobs );

		return _matcher;
	}

	private static Matcher _matcher = null;

	private bool SyncToPublicRepository()
	{
		string remoteBase = null;
		if ( !dryRun )
		{
			remoteBase = GetR2Base();
			if ( string.IsNullOrEmpty( remoteBase ) )
			{
				return false;
			}
		}
		else
		{
			Log.Info( "Dry run enabled: skipping R2 uploads and public push" );
		}

		var repositoryRoot = Path.GetFullPath( "." );
		var filteredRepoPath = CreateShallowClone( repositoryRoot );
		if ( string.IsNullOrEmpty( filteredRepoPath ) )
		{
			return false;
		}

		try
		{
			var relativeFilteredPath = GetRelativeWorkingDirectory( filteredRepoPath );
			var uploadedArtifacts = new List<ArtifactFileInfo>();

			// Upload build artifacts from original repository
			if ( !TryUploadBuildArtifacts( repositoryRoot, remoteBase, dryRun, ref uploadedArtifacts ) )
			{
				return false;
			}

			//
			// Start working with the shallow clone
			//

			// Make certain we dont have any stray files in our shallow clone
			if ( !CleanIgnoredFiles( relativeFilteredPath ) )
			{
				return false;
			}

			// Upload LFS tracked files
			var lfsPaths = GetTrackedLfsFiles( relativeFilteredPath );
			if ( lfsPaths is null )
			{
				return false;
			}

			if ( !TryUploadLfsArtifacts( filteredRepoPath, lfsPaths, remoteBase, dryRun, ref uploadedArtifacts ) )
			{
				return false;
			}

			// Get final set of files to keep after filtering out LFS files
			var pathsToKeep = RepoFileFilter()
				.GetResultsInFullPath( filteredRepoPath )
				.Select( fullPath => ToForwardSlash( Path.GetRelativePath( filteredRepoPath, fullPath ) ) )
				.Where( path => !lfsPaths.Contains( path ) )
				.ToHashSet( StringComparer.OrdinalIgnoreCase );

			// Run git-filter-repo to filter out unwanted paths
			if ( !RunFilterRepo( relativeFilteredPath, pathsToKeep ) )
			{
				return false;
			}

			var publicCommitHash = dryRun
				? "000000"
				: PushToPublicRepository( relativeFilteredPath );
			if ( string.IsNullOrEmpty( publicCommitHash ) )
			{
				return false;
			}

			if ( dryRun )
			{
				Log.Info( $"Dry run filtered repository commit hash: {publicCommitHash}" );
				WriteDryRunOutputs( publicCommitHash, uploadedArtifacts, pathsToKeep );
				return true;
			}

			if ( !UploadManifest( publicCommitHash, uploadedArtifacts, remoteBase ) )
			{
				return false;
			}

			return true;
		}
		finally
		{
			try
			{
				Thread.Sleep( 250 ); // Give any pending file handles a moment to close
				Log.Info( "Cleaning up temporary filtered repository..." );
				Directory.Delete( filteredRepoPath, true );
			}
			catch ( Exception ex )
			{
				Log.Warning( $"Failed to clean up temporary directory: {ex.Message}" );
			}
		}
	}

	private static string CreateShallowClone( string repositoryRoot )
	{
		var localFilePath = new Uri( repositoryRoot ).AbsoluteUri;
		var filteredRepoPath = Path.Combine( Path.GetTempPath(), $"sbox-filtered-{Guid.NewGuid()}" );

		Log.Info( "Creating clone for filtering..." );

		if ( Utility.RunProcess( "git", $"clone --shallow-exclude {SHALLOW_EXCLUDE_TAG} \"{localFilePath}\" \"{filteredRepoPath}\"" ) )
		{
			return filteredRepoPath;
		}

		Log.Error( "Failed to create clone" );
		return null;
	}

	private static bool CleanIgnoredFiles( string relativeRepoPath )
	{
		Log.Info( "Removing ignored files from filtered repository..." );
		if ( Utility.RunProcess( "git", "clean -f -x -d", relativeRepoPath ) )
		{
			return true;
		}

		Log.Error( "Failed to remove ignored files from filtered repository" );
		return false;
	}

	private static bool TryUploadBuildArtifacts( string repositoryRoot, string remoteBase, bool skipUpload, ref List<ArtifactFileInfo> artifacts )
	{
		var buildArtifactsRoot = Path.Combine( repositoryRoot, "game", "bin", "win64" );
		if ( !Directory.Exists( buildArtifactsRoot ) )
		{
			Log.Info( $"Build artifacts directory not found, skipping upload: {buildArtifactsRoot}" );
			return true;
		}

		// Inline matcher: include everything, exclude managed root folder and pdbs
		var matcher = new Matcher( StringComparison.OrdinalIgnoreCase, preserveFilterOrder: true );
		matcher.AddInclude( "**/*" );
		matcher.AddExclude( "managed/**" );
		matcher.AddExclude( "**/*.pdb" );

		var filesToUpload = matcher
			.GetResultsInFullPath( buildArtifactsRoot )
			.ToHashSet( StringComparer.OrdinalIgnoreCase );

		if ( filesToUpload.Count == 0 )
		{
			Log.Info( "No build artifacts found to upload" );
			return true;
		}

		Log.Info( $"Found {filesToUpload.Count} build artifact(s) to upload" );

		var candidates = filesToUpload
			.Select( path =>
			{
				var repoRelativePath = Path.GetRelativePath( repositoryRoot, path );
				return (RepoPath: ToForwardSlash( repoRelativePath ), AbsolutePath: path);
			} )
			.ToList();

		return TryUploadArtifacts( candidates, remoteBase, artifacts, "build", skipUpload );
	}

	private static bool TryUploadLfsArtifacts( string repoRoot, IReadOnlyCollection<string> lfsPaths, string remoteBase, bool skipUpload, ref List<ArtifactFileInfo> artifacts )
	{
		if ( lfsPaths.Count == 0 )
		{
			return true;
		}

		var candidates = lfsPaths
			.Where( p => RepoFileFilter().Match( p ).HasMatches )
			.Select( path => (RepoPath: path, AbsolutePath: Path.Combine( repoRoot, path.Replace( '/', Path.DirectorySeparatorChar ) )) )
			.ToList();

		return TryUploadArtifacts( candidates, remoteBase, artifacts, "LFS", skipUpload );
	}

	private static bool RunFilterRepo( string relativeRepoPath, IReadOnlyCollection<string> pathsToKeep )
	{
		Log.Info( "Running git-filter-repo to filter paths..." );

		var filterArgs = new StringBuilder();
		filterArgs.Append( "filter-repo --force" );

		string tempFile = null;
		try
		{
			if ( pathsToKeep is not null && pathsToKeep.Count > 0 )
			{
				tempFile = Path.GetTempFileName();
				var normalizedPaths = new List<string>( pathsToKeep.Count );
				foreach ( var path in pathsToKeep )
				{
					normalizedPaths.Add( path.Replace( '\\', '/' ) );
				}

				File.WriteAllLines( tempFile, normalizedPaths );
				filterArgs.Append( $" --paths-from-file \"{tempFile}\"" );
			}

			foreach ( var rename in RepoFilterPathRenames )
			{
				filterArgs.Append( $" --path-rename {rename.Key}:{rename.Value}" );
			}

			// Reference the original commit, and mark our baseline commit
			var commitCallback = """
				if not commit.parents:
					commit.message = b'Open source release\n\nThis commit imports the C# engine code and game files, excluding C++ source code.'
					commit.author_name = b's&box team'
					commit.author_email = b'sboxbot@facepunch.com'
					commit.committer_name = b's&box team'
					commit.committer_email = b'sboxbot@facepunch.com'
					commit.message += b'\n\n[Source-Commit: ' + commit.original_id + b']\n'
				""";
			filterArgs.Append( $" --commit-callback \"{commitCallback}\"" );

			if ( Utility.RunProcess( "git", filterArgs.ToString(), relativeRepoPath ) )
			{
				return true;
			}

			Log.Error( "Failed to filter repository" );
			return false;
		}
		finally
		{
			if ( tempFile is not null && File.Exists( tempFile ) )
			{
				File.Delete( tempFile );
			}
		}
	}

	private string PushToPublicRepository( string relativeRepoPath )
	{
		Log.Info( "Pushing filtered repository to public..." );

		var token = Environment.GetEnvironmentVariable( "SYNC_GITHUB_TOKEN" );
		if ( string.IsNullOrEmpty( token ) )
		{
			Log.Error( "SYNC_GITHUB_TOKEN environment variable not set" );
			return null;
		}

		var publicUrl = $"https://x-access-token:{token}@github.com/{PUBLIC_REPO}.git";
		if ( !Utility.RunProcess( "git", $"remote add public \"{publicUrl}\"", relativeRepoPath ) )
		{
			Log.Warning( "Failed to add remote (may already exist), attempting to update URL" );
			if ( !Utility.RunProcess( "git", $"remote set-url public \"{publicUrl}\"", relativeRepoPath ) )
			{
				Log.Error( "Failed to configure public remote" );
				return null;
			}
		}

		if ( !Utility.RunProcess( "git", $"push public {PUBLIC_BRANCH} --force", relativeRepoPath ) )
		{
			Log.Error( "Failed to push to public repository" );
			return null;
		}

		string publicCommitHash = null;
		if ( !Utility.RunProcess( "git", "rev-parse HEAD", relativeRepoPath, onDataReceived: ( _, e ) =>
		{
			if ( !string.IsNullOrWhiteSpace( e.Data ) )
			{
				publicCommitHash ??= e.Data.Trim();
			}
		} ) )
		{
			Log.Error( "Failed to retrieve public commit hash" );
			return null;
		}

		if ( string.IsNullOrWhiteSpace( publicCommitHash ) )
		{
			Log.Error( "Public commit hash was empty" );
			return null;
		}

		Log.Info( $"Public repository commit hash: {publicCommitHash}" );

		return publicCommitHash;
	}

	private static HashSet<string> GetTrackedLfsFiles( string relativeRepoPath )
	{
		var trackedFiles = new HashSet<string>( StringComparer.OrdinalIgnoreCase );
		if ( !Utility.RunProcess( "git", "lfs ls-files --name-only", relativeRepoPath, onDataReceived: ( _, e ) =>
		{
			if ( !string.IsNullOrWhiteSpace( e.Data ) )
			{
				trackedFiles.Add( ToForwardSlash( e.Data.Trim() ) );
			}
		} ) )
		{
			Log.Error( "Failed to list LFS tracked files" );
			return null;
		}

		Log.Info( trackedFiles.Count == 0
			? "No LFS tracked files eligible for upload"
			: $"Found {trackedFiles.Count} LFS tracked files eligible for upload" );

		return trackedFiles;
	}

	private static string ToForwardSlash( string path )
	{
		return path.Replace( '\\', '/' );
	}

	private void WriteDryRunOutputs( string commitHash, IReadOnlyList<ArtifactFileInfo> artifacts, IReadOnlyCollection<string> pathsToKeep )
	{
		var workingDirectory = Directory.GetCurrentDirectory();
		var manifestPath = Path.Combine( workingDirectory, "public-sync-manifest.dryrun.json" );
		var manifest = new ArtifactManifest
		{
			Commit = commitHash,
			Timestamp = DateTime.UtcNow.ToString( "o" ),
			Files = artifacts is null ? new List<ArtifactFileInfo>() : new List<ArtifactFileInfo>( artifacts )
		};

		var manifestJson = JsonSerializer.Serialize( manifest, new JsonSerializerOptions { WriteIndented = true } );
		File.WriteAllText( manifestPath, manifestJson );

		var pathsOutputPath = Path.Combine( workingDirectory, "public-sync-paths.dryrun.txt" );
		var pathsSequence = pathsToKeep ?? Array.Empty<string>();
		File.WriteAllLines( pathsOutputPath, pathsSequence );

		Log.Info( $"Dry run manifest written to {manifestPath}" );
		Log.Info( $"Dry run paths list written to {pathsOutputPath}" );
	}

	private static bool TryUploadArtifacts( IReadOnlyCollection<(string RepoPath, string AbsolutePath)> candidates, string remoteBase, List<ArtifactFileInfo> artifacts, string artifactLabel, bool skipUpload )
	{
		if ( candidates.Count == 0 )
		{
			Log.Info( $"No {artifactLabel} artifacts found to upload" );
			return true;
		}

		var uniqueUploads = new Dictionary<string, (string AbsolutePath, ArtifactFileInfo Artifact)>( StringComparer.OrdinalIgnoreCase );

		foreach ( var (repoPath, absolutePath) in candidates )
		{
			var repoPathNormalized = ToForwardSlash( repoPath );

			if ( !File.Exists( absolutePath ) )
			{
				Log.Error( $"Artifact not found on disk: {repoPathNormalized}" );
				return false;
			}

			var fileInfo = new FileInfo( absolutePath );
			var sha256 = Utility.CalculateSha256( absolutePath );
			var artifact = new ArtifactFileInfo
			{
				Path = repoPathNormalized,
				Sha256 = sha256,
				Size = fileInfo.Length
			};

			artifacts.Add( artifact );
			if ( !uniqueUploads.TryAdd( sha256, (absolutePath, artifact) ) )
			{
				Log.Info( $"Skipping upload for {repoPathNormalized} (hash {sha256} already queued for upload)" );
			}
		}

		if ( uniqueUploads.Count == 0 )
		{
			Log.Info( $"No unique {artifactLabel} artifacts to upload" );
			return true;
		}

		if ( skipUpload )
		{
			Log.Info( $"Dry run: skipping upload for {uniqueUploads.Count} unique {artifactLabel} artifacts" );
			return true;
		}

		var maxParallelUploads = Math.Max( 1, Math.Min( MAX_PARALLEL_UPLOADS, Environment.ProcessorCount ) );
		Log.Info( $"Uploading {uniqueUploads.Count} unique {artifactLabel} artifacts (up to {maxParallelUploads} concurrent uploads)..." );

		var failedUploads = new ConcurrentBag<string>();
		Parallel.ForEach( uniqueUploads, new ParallelOptions { MaxDegreeOfParallelism = maxParallelUploads }, kvp =>
		{
			var (absolutePath, artifact) = kvp.Value;
			if ( !UploadArtifactFile( absolutePath, artifact, remoteBase ) )
			{
				Log.Error( $"Failed to upload {artifactLabel} artifact: {artifact.Path}" );
				failedUploads.Add( artifact.Path );
			}
		} );

		if ( !failedUploads.IsEmpty )
		{
			Log.Error( $"Failed to upload {failedUploads.Count} {artifactLabel} artifact(s)" );
			return false;
		}

		Log.Info( $"Uploaded {uniqueUploads.Count} unique {artifactLabel} artifacts" );
		return true;
	}

	private static bool UploadArtifactFile( string localPath, ArtifactFileInfo artifact, string remoteBase )
	{
		var remotePath = $"{remoteBase}/artifacts/{artifact.Sha256}";
		var sizeLabel = $" ({Utility.FormatSize( artifact.Size )})";
		Log.Info( $"Uploading {artifact.Sha256}{sizeLabel}..." );
		return Utility.RunProcess( "rclone", $"copyto \"{localPath}\" \"{remotePath}\" --ignore-existing -q", timeoutMs: 600000 );
	}

	private static bool UploadManifest( string commitHash, IReadOnlyList<ArtifactFileInfo> artifacts, string remoteBase )
	{
		var files = artifacts is null ? new List<ArtifactFileInfo>() : new List<ArtifactFileInfo>( artifacts );
		var manifest = new ArtifactManifest
		{
			Commit = commitHash,
			Timestamp = DateTime.UtcNow.ToString( "o" ),
			Files = files
		};

		var manifestJson = JsonSerializer.Serialize( manifest, new JsonSerializerOptions { WriteIndented = true } );

		var manifestPath = Path.Combine( Path.GetTempPath(), $"{commitHash}.json" );
		File.WriteAllText( manifestPath, manifestJson );

		try
		{
			Log.Info( $"Uploading manifest: {commitHash}.json with {manifest.Files.Count} files" );
			var remotePath = $"{remoteBase}/manifests/{commitHash}.json";
			if ( !Utility.RunProcess( "rclone", $"copyto \"{manifestPath}\" \"{remotePath}\"", timeoutMs: 60000 ) )
			{
				Log.Error( "Failed to upload manifest file" );
				return false;
			}
		}
		finally
		{
			if ( File.Exists( manifestPath ) )
			{
				File.Delete( manifestPath );
			}
		}

		return true;
	}

	private static string GetRelativeWorkingDirectory( string absolutePath )
	{
		var repoRoot = Directory.GetCurrentDirectory();
		var relativePath = Path.GetRelativePath( repoRoot, absolutePath );
		return string.IsNullOrEmpty( relativePath ) ? "." : relativePath;
	}

	private static string GetR2Base()
	{
		var r2AccessKeyId = Environment.GetEnvironmentVariable( "SYNC_R2_ACCESS_KEY_ID" );
		var r2SecretAccessKey = Environment.GetEnvironmentVariable( "SYNC_R2_SECRET_ACCESS_KEY" );
		var r2Bucket = Environment.GetEnvironmentVariable( "SYNC_R2_BUCKET" );
		var r2Endpoint = Environment.GetEnvironmentVariable( "SYNC_R2_ENDPOINT" );

		if ( string.IsNullOrEmpty( r2AccessKeyId ) || string.IsNullOrEmpty( r2SecretAccessKey ) ||
			 string.IsNullOrEmpty( r2Bucket ) || string.IsNullOrEmpty( r2Endpoint ) )
		{
			Log.Error( "R2 credentials not properly configured in environment variables" );
			return null;
		}

		return $":s3,bucket={r2Bucket},provider=Cloudflare,access_key_id={r2AccessKeyId},secret_access_key={r2SecretAccessKey},endpoint='{r2Endpoint}':";
	}
}