diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..a8478469 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,16 @@ +# Auto detect text files +* text=auto + +# Anything under game (content) force crlf for consistent content hashes +game/** text=auto eol=crlf +# Except c# files, auto that +game/**/*.cs text=auto + +# +# GitHub linguist overrides (https://github.com/github-linguist/linguist/blob/main/docs/overrides.md) +# Lets GitHub know the proper languages, what code is generated, third party, etc. for better stats/diffs +# +*.h linguist-language=C++ +*.shader linguist-language=HLSL +*.fxc linguist-language=HLSL +src/thirdparty/* linguist-vendored diff --git a/.gitignore b/.gitignore index 92f321ec..46161c47 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,10 @@ * !*/ +# Git +!.gitignore +!.gitattributes + # Source files !*.sln !*.csproj diff --git a/engine/Tools/SboxBuild/Program.cs b/engine/Tools/SboxBuild/Program.cs index 5dee5f26..a7e30310 100644 --- a/engine/Tools/SboxBuild/Program.cs +++ b/engine/Tools/SboxBuild/Program.cs @@ -23,6 +23,8 @@ internal class Program AddBuildShadersStep( rootCommand ); AddGenerateSolutionsStep( rootCommand ); + AddSyncPublicRepo( rootCommand ); + AddPullRequestPipeline( rootCommand ); AddDeployPipeline( rootCommand ); @@ -191,4 +193,25 @@ internal class Program rootCommand.Add( generateSolutionsCommand ); } + + private static void AddSyncPublicRepo( RootCommand rootCommand ) + { + var syncCommand = new Command( "sync-public-repo", "Perform a dry run of the sync step" ); + + var option = new Option( + "--dry-run", + description: "Whether to perform a dry run", + getDefaultValue: () => true ); + + syncCommand.AddOption( option ); + + syncCommand.SetHandler( ( bool dryRun ) => + { + var step = new SyncPublicRepo( "Sync Public Repo", dryRun ); + ExitCode result = step.Run(); + Environment.ExitCode = (int)result; + }, option ); + + rootCommand.Add( syncCommand ); + } } diff --git a/engine/Tools/SboxBuild/SboxBuild.csproj b/engine/Tools/SboxBuild/SboxBuild.csproj index 1cdbb0aa..f7bc4424 100644 --- a/engine/Tools/SboxBuild/SboxBuild.csproj +++ b/engine/Tools/SboxBuild/SboxBuild.csproj @@ -17,6 +17,7 @@ + diff --git a/engine/Tools/SboxBuild/Steps/SyncPublicRepo.cs b/engine/Tools/SboxBuild/Steps/SyncPublicRepo.cs index 1a6921dc..329b7015 100644 --- a/engine/Tools/SboxBuild/Steps/SyncPublicRepo.cs +++ b/engine/Tools/SboxBuild/Steps/SyncPublicRepo.cs @@ -1,4 +1,8 @@ +using Microsoft.Extensions.FileSystemGlobbing; +using System; using System.Collections.Concurrent; +using System.IO; +using System.Linq; using System.Text; using System.Text.Json; using System.Text.Json.Serialization; @@ -30,43 +34,26 @@ internal record ArtifactManifest public List Files { get; init; } } -internal record PublicSyncResult( string CommitHash, List Artifacts ); - /// /// Syncs the master branch to the public repository by filtering specific paths /// -internal class SyncPublicRepo( string name ) : Step( name ) +internal class SyncPublicRepo( string name, bool dryRun = false ) : Step( name ) { private const string PUBLIC_REPO = "Facepunch/sbox-public"; private const string PUBLIC_BRANCH = "master"; + private const string SHALLOW_EXCLUDE_TAG = "public-history-root"; private const int MAX_PARALLEL_UPLOADS = 32; + protected override ExitCode RunInternal() { try { - if ( !TryGetRemoteBase( out var remoteBase ) ) + var success = SyncToPublicRepository(); + if ( !success ) { return ExitCode.Failure; } - var syncResult = SyncToPublicRepository( remoteBase ); - - if ( syncResult is null || string.IsNullOrEmpty( syncResult.CommitHash ) ) - { - Log.Error( "Failed to sync to public repository" ); - return ExitCode.Failure; - } - - if ( !UploadManifest( syncResult.CommitHash, syncResult.Artifacts, remoteBase ) ) - { - Log.Error( "Failed to upload manifest" ); - return ExitCode.Failure; - } - - Environment.SetEnvironmentVariable( "PUBLIC_COMMIT_HASH", syncResult.CommitHash ); - - Log.Info( $"Successfully synced to public repository. Public commit: {syncResult.CommitHash}" ); - return ExitCode.Success; } catch ( Exception ex ) @@ -76,296 +63,197 @@ internal class SyncPublicRepo( string name ) : Step( name ) } } - private static bool TryGetRemoteBase( out string remoteBase ) + private static readonly string[] RepoFilterPathIncludeGlobs = { - remoteBase = null; + "engine/**", + "game/**", + ".editorconfig", + "public/**" + }; - var r2AccessKeyId = Environment.GetEnvironmentVariable( "SYNC_R2_ACCESS_KEY_ID" ); - var r2SecretAccessKey = Environment.GetEnvironmentVariable( "SYNC_R2_SECRET_ACCESS_KEY" ); - var r2Bucket = Environment.GetEnvironmentVariable( "SYNC_R2_BUCKET" ); - var r2Endpoint = Environment.GetEnvironmentVariable( "SYNC_R2_ENDPOINT" ); + private static readonly string[] RepoFilterPathExcludeGlobs = + { + "**/*.pdb", + "game/core/shaders/**" + }; - if ( string.IsNullOrEmpty( r2AccessKeyId ) || string.IsNullOrEmpty( r2SecretAccessKey ) || - string.IsNullOrEmpty( r2Bucket ) || string.IsNullOrEmpty( r2Endpoint ) ) + private static readonly string[] RepoFilterShaderWhitelistGlobs = + { + "game/core/shaders/**/vr_*", + "game/core/shaders/**/*.shader_c", + "game/core/shaders/common.fxc", + "game/core/shaders/common_samplers.fxc", + "game/core/shaders/descriptor_set_support.fxc", + "game/core/shaders/system.fxc", + "game/core/shaders/tiled_culling.hlsl" + }; + + private static readonly Dictionary RepoFilterPathRenames = new() + { + { "public/.gitignore", ".gitignore" }, + { "public/.gitattributes", ".gitattributes" }, + { "public/README.md", "README.md" }, + { "public/LICENSE.md", "LICENSE.md" }, + { "public/CONTRIBUTING.md", "CONTRIBUTING.md" }, + { "public/Bootstrap.bat", "Bootstrap.bat" } + }; + + private static Matcher RepoFileFilter() + { + if ( _matcher is not null ) { - Log.Error( "R2 credentials not properly configured in environment variables" ); - return false; + return _matcher; } - remoteBase = BuildRcloneRemoteBase( r2Bucket, r2AccessKeyId, r2SecretAccessKey, r2Endpoint ); - return true; + // Ordered since we first include everything, then exclude, then re-include specific files + _matcher = new Matcher( StringComparison.OrdinalIgnoreCase, preserveFilterOrder: true ); + + _matcher.AddIncludePatterns( RepoFilterPathIncludeGlobs ); + + _matcher.AddExcludePatterns( RepoFilterPathExcludeGlobs ); + + _matcher.AddIncludePatterns( RepoFilterShaderWhitelistGlobs ); + + return _matcher; } - private PublicSyncResult SyncToPublicRepository( string remoteBase ) + private static Matcher _matcher = null; + + private bool SyncToPublicRepository() { + string remoteBase = null; + if ( !dryRun ) + { + remoteBase = GetR2Base(); + if ( string.IsNullOrEmpty( remoteBase ) ) + { + return false; + } + } + else + { + Log.Info( "Dry run enabled: skipping R2 uploads and public push" ); + } + var repositoryRoot = Path.GetFullPath( "." ); - var localFilePath = new Uri( repositoryRoot ).AbsoluteUri; - var filteredRepoPath = Path.Combine( Path.GetTempPath(), $"sbox-filtered-{Guid.NewGuid()}" ); + var filteredRepoPath = CreateShallowClone( repositoryRoot ); + if ( string.IsNullOrEmpty( filteredRepoPath ) ) + { + return false; + } try { - Log.Info( "Creating clone for filtering..." ); - - if ( !Utility.RunProcess( "git", $"clone --shallow-exclude public-history-root \"{localFilePath}\" \"{filteredRepoPath}\"" ) ) - { - Log.Error( "Failed to create clone" ); - return null; - } - - Log.Info( "Running git-filter-repo to filter paths..." ); - - var filterArgs = BuildFilterRepoArguments(); - var relativeFilteredPath = GetRelativeWorkingDirectory( filteredRepoPath ); - - if ( !Utility.RunProcess( "git", filterArgs, relativeFilteredPath ) ) - { - Log.Error( "Failed to filter repository" ); - return null; - } - - var lfsTrackedFiles = GetLfsTrackedFiles( relativeFilteredPath ); - if ( lfsTrackedFiles is null ) - { - return null; - } - var uploadedArtifacts = new List(); - if ( lfsTrackedFiles.Count > 0 ) + // Upload build artifacts from original repository + if ( !TryUploadBuildArtifacts( repositoryRoot, remoteBase, dryRun, ref uploadedArtifacts ) ) { - Log.Info( $"Found {lfsTrackedFiles.Count} LFS tracked files to upload" ); - if ( !TryUploadLfsArtifacts( filteredRepoPath, lfsTrackedFiles, remoteBase, uploadedArtifacts ) ) - { - return null; - } - - if ( !RemovePathsFromRepo( lfsTrackedFiles, relativeFilteredPath ) ) - { - return null; - } - - Log.Info( $"Removed {lfsTrackedFiles.Count} LFS tracked files from filtered repository" ); - } - else - { - Log.Info( "No LFS tracked files found in filtered repository" ); - } - - if ( !TryUploadBuildArtifacts( repositoryRoot, remoteBase, uploadedArtifacts ) ) - { - return null; - } - - Log.Info( "Pushing filtered repository to public..." ); - - var token = Environment.GetEnvironmentVariable( "SYNC_GITHUB_TOKEN" ); - if ( string.IsNullOrEmpty( token ) ) - { - Log.Error( "SYNC_GITHUB_TOKEN environment variable not set" ); - return null; - } - - var publicUrl = $"https://x-access-token:{token}@github.com/{PUBLIC_REPO}.git"; - if ( !Utility.RunProcess( "git", $"remote add public \"{publicUrl}\"", relativeFilteredPath ) ) - { - Log.Warning( "Failed to add remote (may already exist), attempting to update URL" ); - if ( !Utility.RunProcess( "git", $"remote set-url public \"{publicUrl}\"", relativeFilteredPath ) ) - { - Log.Error( "Failed to configure public remote" ); - return null; - } - } - - if ( !Utility.RunProcess( "git", $"push public {PUBLIC_BRANCH} --force", relativeFilteredPath ) ) - { - Log.Error( "Failed to push to public repository" ); - return null; - } - - string publicCommitHash = null; - if ( !Utility.RunProcess( "git", "rev-parse HEAD", relativeFilteredPath, onDataReceived: ( _, e ) => - { - if ( !string.IsNullOrWhiteSpace( e.Data ) ) - { - publicCommitHash ??= e.Data.Trim(); - } - } ) ) - { - Log.Error( "Failed to retrieve public commit hash" ); - return null; - } - - if ( string.IsNullOrWhiteSpace( publicCommitHash ) ) - { - Log.Error( "Public commit hash was empty" ); - return null; - } - - Log.Info( $"Public repository commit hash: {publicCommitHash}" ); - - return new PublicSyncResult( publicCommitHash, uploadedArtifacts ); - } - finally - { - TryDeleteDirectory( filteredRepoPath ); - } - } - - private static string BuildFilterRepoArguments() - { - var filterArgs = new StringBuilder(); - filterArgs.Append( "filter-repo --force " ); - - var pathsToKeep = new[] - { - "engine", - "game", - ".editorconfig", - ".gitattributes", - "public/" - }; - - foreach ( var path in pathsToKeep ) - { - filterArgs.Append( $"--path {path} " ); - } - - var renames = new Dictionary - { - { "public/.gitignore", ".gitignore" }, - { "public/README.md", "README.md" }, - { "public/LICENSE.md", "LICENSE.md" }, - { "public/CONTRIBUTING.md", "CONTRIBUTING.md" }, - { "public/Bootstrap.bat", "Bootstrap.bat" } - }; - - foreach ( var rename in renames ) - { - filterArgs.Append( $"--path-rename {rename.Key}:{rename.Value} " ); - } - - // Reference the original commit, and mark our baseline commit - var commitCallback = """ - if not commit.parents: - commit.message = b'Open source release\n\nThis commit imports the C# engine code and game files, excluding C++ source code.' - commit.author_name = b's&box team' - commit.author_email = b'sboxbot@facepunch.com' - commit.committer_name = b's&box team' - commit.committer_email = b'sboxbot@facepunch.com' - commit.message += b'\n\n[Source-Commit: ' + commit.original_id + b']\n' - """; - - const string filenameCallback = "return filename if (filename is None or (not filename.endswith(b'.pdb') and (b'game/core/shaders/' not in filename or (lambda base: base.startswith(b'vr_') or base.endswith(b'shader_c') or base in {b'common.fxc', b'common_samplers.fxc', b'descriptor_set_support.fxc', b'system.fxc', b'tiled_culling.hlsl'})(filename.split(b'/')[-1])))) else None"; - filterArgs.Append( $"--filename-callback \"{filenameCallback}\" " ); - filterArgs.Append( $"--commit-callback \"{commitCallback}\"" ); - - return filterArgs.ToString(); - } - - private static List GetLfsTrackedFiles( string relativeRepoPath ) - { - var trackedFiles = new List(); - var uniqueFiles = new HashSet( StringComparer.OrdinalIgnoreCase ); - - if ( !Utility.RunProcess( "git", "lfs ls-files --name-only", relativeRepoPath, onDataReceived: ( _, e ) => - { - if ( string.IsNullOrWhiteSpace( e.Data ) ) - { - return; - } - - var path = e.Data.Trim(); - if ( uniqueFiles.Add( path ) ) - { - trackedFiles.Add( path ); - } - } ) ) - { - Log.Error( "Failed to list LFS tracked files" ); - return null; - } - - trackedFiles.Sort( StringComparer.OrdinalIgnoreCase ); - return trackedFiles; - } - - private static bool TryUploadLfsArtifacts( string repoRoot, IReadOnlyCollection lfsPaths, string remoteBase, List artifacts ) - { - if ( artifacts is null ) - { - throw new ArgumentNullException( nameof( artifacts ) ); - } - - var uniqueUploads = new Dictionary( StringComparer.OrdinalIgnoreCase ); - - foreach ( var lfsPath in lfsPaths ) - { - var normalizedPath = NormalizeRepoPath( lfsPath ); - - if ( string.Equals( Path.GetExtension( normalizedPath ), ".pdb", StringComparison.OrdinalIgnoreCase ) ) - { - Log.Info( $"Skipping LFS artifact with excluded extension: {lfsPath}" ); - continue; - } - - var absolutePath = Path.Combine( repoRoot, normalizedPath ); - - if ( !File.Exists( absolutePath ) ) - { - Log.Error( $"LFS file not found on disk: {lfsPath}" ); return false; } - var fileInfo = new FileInfo( absolutePath ); - var sha256 = Utility.CalculateSha256( absolutePath ); + // + // Start working with the shallow clone + // - var artifact = new ArtifactFileInfo + // Make certain we dont have any stray files in our shallow clone + if ( !CleanIgnoredFiles( relativeFilteredPath ) ) { - Path = lfsPath.Replace( '\\', '/' ), - Sha256 = sha256, - Size = fileInfo.Length - }; + return false; + } - artifacts.Add( artifact ); - - if ( !uniqueUploads.TryAdd( sha256, (absolutePath, artifact) ) ) + // Upload LFS tracked files + var lfsPaths = GetTrackedLfsFiles( relativeFilteredPath ); + if ( lfsPaths is null ) { - Log.Info( $"Skipping upload for {lfsPath} (already uploaded hash {sha256})" ); + return false; + } + + if ( !TryUploadLfsArtifacts( filteredRepoPath, lfsPaths, remoteBase, dryRun, ref uploadedArtifacts ) ) + { + return false; + } + + // Get final set of files to keep after filtering out LFS files + var pathsToKeep = RepoFileFilter() + .GetResultsInFullPath( filteredRepoPath ) + .Select( fullPath => ToForwardSlash( Path.GetRelativePath( filteredRepoPath, fullPath ) ) ) + .Where( path => !lfsPaths.Contains( path ) ) + .ToHashSet( StringComparer.OrdinalIgnoreCase ); + + // Run git-filter-repo to filter out unwanted paths + if ( !RunFilterRepo( relativeFilteredPath, pathsToKeep ) ) + { + return false; + } + + var publicCommitHash = dryRun + ? "000000" + : PushToPublicRepository( relativeFilteredPath ); + if ( string.IsNullOrEmpty( publicCommitHash ) ) + { + return false; + } + + if ( dryRun ) + { + Log.Info( $"Dry run filtered repository commit hash: {publicCommitHash}" ); + WriteDryRunOutputs( publicCommitHash, uploadedArtifacts, pathsToKeep ); + return true; + } + + if ( !UploadManifest( publicCommitHash, uploadedArtifacts, remoteBase ) ) + { + return false; + } + + return true; + } + finally + { + try + { + Thread.Sleep( 250 ); // Give any pending file handles a moment to close + Log.Info( "Cleaning up temporary filtered repository..." ); + Directory.Delete( filteredRepoPath, true ); + } + catch ( Exception ex ) + { + Log.Warning( $"Failed to clean up temporary directory: {ex.Message}" ); } } + } - if ( uniqueUploads.Count == 0 ) + private static string CreateShallowClone( string repositoryRoot ) + { + var localFilePath = new Uri( repositoryRoot ).AbsoluteUri; + var filteredRepoPath = Path.Combine( Path.GetTempPath(), $"sbox-filtered-{Guid.NewGuid()}" ); + + Log.Info( "Creating clone for filtering..." ); + + if ( Utility.RunProcess( "git", $"clone --shallow-exclude {SHALLOW_EXCLUDE_TAG} \"{localFilePath}\" \"{filteredRepoPath}\"" ) ) + { + return filteredRepoPath; + } + + Log.Error( "Failed to create clone" ); + return null; + } + + private static bool CleanIgnoredFiles( string relativeRepoPath ) + { + Log.Info( "Removing ignored files from filtered repository..." ); + if ( Utility.RunProcess( "git", "clean -f -x -d", relativeRepoPath ) ) { - Log.Info( "No unique LFS artifacts to upload" ); return true; } - var maxParallelUploads = Math.Max( 1, Math.Min( MAX_PARALLEL_UPLOADS, Environment.ProcessorCount ) ); - Log.Info( $"Uploading {uniqueUploads.Count} unique LFS artifacts (up to {maxParallelUploads} concurrent uploads)..." ); - - var failedUploads = new ConcurrentBag(); - Parallel.ForEach( uniqueUploads, new ParallelOptions { MaxDegreeOfParallelism = maxParallelUploads }, kvp => - { - var (absolutePath, artifact) = kvp.Value; - if ( !UploadArtifactFile( absolutePath, artifact, remoteBase ) ) - { - Log.Error( $"Failed to upload LFS artifact: {artifact.Path}" ); - failedUploads.Add( artifact.Path ); - } - } ); - - if ( !failedUploads.IsEmpty ) - { - Log.Error( $"Failed to upload {failedUploads.Count} artifact(s)" ); - return false; - } - - Log.Info( $"Uploaded {uniqueUploads.Count} unique LFS artifacts" ); - return true; + Log.Error( "Failed to remove ignored files from filtered repository" ); + return false; } - private static bool TryUploadBuildArtifacts( string repositoryRoot, string remoteBase, List artifacts ) + private static bool TryUploadBuildArtifacts( string repositoryRoot, string remoteBase, bool skipUpload, ref List artifacts ) { var buildArtifactsRoot = Path.Combine( repositoryRoot, "game", "bin", "win64" ); if ( !Directory.Exists( buildArtifactsRoot ) ) @@ -374,31 +262,15 @@ internal class SyncPublicRepo( string name ) : Step( name ) return true; } - var filesToUpload = new List(); - try - { - foreach ( var filePath in Directory.EnumerateFiles( buildArtifactsRoot, "*", SearchOption.AllDirectories ) ) - { - if ( string.Equals( Path.GetExtension( filePath ), ".pdb", StringComparison.OrdinalIgnoreCase ) ) - { - continue; - } + // Inline matcher: include everything, exclude managed root folder and pdbs + var matcher = new Matcher( StringComparison.OrdinalIgnoreCase, preserveFilterOrder: true ); + matcher.AddInclude( "**/*" ); + matcher.AddExclude( "managed/**" ); + matcher.AddExclude( "**/*.pdb" ); - var relativeToBuild = Path.GetRelativePath( buildArtifactsRoot, filePath ); - var segments = relativeToBuild.Split( new[] { Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar }, System.StringSplitOptions.RemoveEmptyEntries ); - if ( segments.Length > 0 && string.Equals( segments[0], "managed", StringComparison.OrdinalIgnoreCase ) ) - { - continue; - } - - filesToUpload.Add( filePath ); - } - } - catch ( Exception ex ) - { - Log.Error( $"Failed to enumerate build artifacts: {ex.Message}" ); - return false; - } + var filesToUpload = matcher + .GetResultsInFullPath( buildArtifactsRoot ) + .ToHashSet( StringComparer.OrdinalIgnoreCase ); if ( filesToUpload.Count == 0 ) { @@ -408,31 +280,240 @@ internal class SyncPublicRepo( string name ) : Step( name ) Log.Info( $"Found {filesToUpload.Count} build artifact(s) to upload" ); + var candidates = filesToUpload + .Select( path => + { + var repoRelativePath = Path.GetRelativePath( repositoryRoot, path ); + return (RepoPath: ToForwardSlash( repoRelativePath ), AbsolutePath: path); + } ) + .ToList(); + + return TryUploadArtifacts( candidates, remoteBase, artifacts, "build", skipUpload ); + } + + private static bool TryUploadLfsArtifacts( string repoRoot, IReadOnlyCollection lfsPaths, string remoteBase, bool skipUpload, ref List artifacts ) + { + if ( lfsPaths.Count == 0 ) + { + return true; + } + + var candidates = lfsPaths + .Where( p => RepoFileFilter().Match( p ).HasMatches ) + .Select( path => (RepoPath: path, AbsolutePath: Path.Combine( repoRoot, path.Replace( '/', Path.DirectorySeparatorChar ) )) ) + .ToList(); + + return TryUploadArtifacts( candidates, remoteBase, artifacts, "LFS", skipUpload ); + } + + private static bool RunFilterRepo( string relativeRepoPath, IReadOnlyCollection pathsToKeep ) + { + Log.Info( "Running git-filter-repo to filter paths..." ); + + var filterArgs = new StringBuilder(); + filterArgs.Append( "filter-repo --force" ); + + string tempFile = null; + try + { + if ( pathsToKeep is not null && pathsToKeep.Count > 0 ) + { + tempFile = Path.GetTempFileName(); + var normalizedPaths = new List( pathsToKeep.Count ); + foreach ( var path in pathsToKeep ) + { + normalizedPaths.Add( path.Replace( '\\', '/' ) ); + } + + File.WriteAllLines( tempFile, normalizedPaths ); + filterArgs.Append( $" --paths-from-file \"{tempFile}\"" ); + } + + foreach ( var rename in RepoFilterPathRenames ) + { + filterArgs.Append( $" --path-rename {rename.Key}:{rename.Value}" ); + } + + // Reference the original commit, and mark our baseline commit + var commitCallback = """ + if not commit.parents: + commit.message = b'Open source release\n\nThis commit imports the C# engine code and game files, excluding C++ source code.' + commit.author_name = b's&box team' + commit.author_email = b'sboxbot@facepunch.com' + commit.committer_name = b's&box team' + commit.committer_email = b'sboxbot@facepunch.com' + commit.message += b'\n\n[Source-Commit: ' + commit.original_id + b']\n' + """; + filterArgs.Append( $" --commit-callback \"{commitCallback}\"" ); + + if ( Utility.RunProcess( "git", filterArgs.ToString(), relativeRepoPath ) ) + { + return true; + } + + Log.Error( "Failed to filter repository" ); + return false; + } + finally + { + if ( tempFile is not null && File.Exists( tempFile ) ) + { + File.Delete( tempFile ); + } + } + } + + private string PushToPublicRepository( string relativeRepoPath ) + { + Log.Info( "Pushing filtered repository to public..." ); + + var token = Environment.GetEnvironmentVariable( "SYNC_GITHUB_TOKEN" ); + if ( string.IsNullOrEmpty( token ) ) + { + Log.Error( "SYNC_GITHUB_TOKEN environment variable not set" ); + return null; + } + + var publicUrl = $"https://x-access-token:{token}@github.com/{PUBLIC_REPO}.git"; + if ( !Utility.RunProcess( "git", $"remote add public \"{publicUrl}\"", relativeRepoPath ) ) + { + Log.Warning( "Failed to add remote (may already exist), attempting to update URL" ); + if ( !Utility.RunProcess( "git", $"remote set-url public \"{publicUrl}\"", relativeRepoPath ) ) + { + Log.Error( "Failed to configure public remote" ); + return null; + } + } + + if ( !Utility.RunProcess( "git", $"push public {PUBLIC_BRANCH} --force", relativeRepoPath ) ) + { + Log.Error( "Failed to push to public repository" ); + return null; + } + + string publicCommitHash = null; + if ( !Utility.RunProcess( "git", "rev-parse HEAD", relativeRepoPath, onDataReceived: ( _, e ) => + { + if ( !string.IsNullOrWhiteSpace( e.Data ) ) + { + publicCommitHash ??= e.Data.Trim(); + } + } ) ) + { + Log.Error( "Failed to retrieve public commit hash" ); + return null; + } + + if ( string.IsNullOrWhiteSpace( publicCommitHash ) ) + { + Log.Error( "Public commit hash was empty" ); + return null; + } + + Log.Info( $"Public repository commit hash: {publicCommitHash}" ); + + return publicCommitHash; + } + + private static HashSet GetTrackedLfsFiles( string relativeRepoPath ) + { + var trackedFiles = new HashSet( StringComparer.OrdinalIgnoreCase ); + if ( !Utility.RunProcess( "git", "lfs ls-files --name-only", relativeRepoPath, onDataReceived: ( _, e ) => + { + if ( !string.IsNullOrWhiteSpace( e.Data ) ) + { + trackedFiles.Add( ToForwardSlash( e.Data.Trim() ) ); + } + } ) ) + { + Log.Error( "Failed to list LFS tracked files" ); + return null; + } + + Log.Info( trackedFiles.Count == 0 + ? "No LFS tracked files eligible for upload" + : $"Found {trackedFiles.Count} LFS tracked files eligible for upload" ); + + return trackedFiles; + } + + private static string ToForwardSlash( string path ) + { + return path.Replace( '\\', '/' ); + } + + private void WriteDryRunOutputs( string commitHash, IReadOnlyList artifacts, IReadOnlyCollection pathsToKeep ) + { + var workingDirectory = Directory.GetCurrentDirectory(); + var manifestPath = Path.Combine( workingDirectory, "public-sync-manifest.dryrun.json" ); + var manifest = new ArtifactManifest + { + Commit = commitHash, + Timestamp = DateTime.UtcNow.ToString( "o" ), + Files = artifacts is null ? new List() : new List( artifacts ) + }; + + var manifestJson = JsonSerializer.Serialize( manifest, new JsonSerializerOptions { WriteIndented = true } ); + File.WriteAllText( manifestPath, manifestJson ); + + var pathsOutputPath = Path.Combine( workingDirectory, "public-sync-paths.dryrun.txt" ); + var pathsSequence = pathsToKeep ?? Array.Empty(); + File.WriteAllLines( pathsOutputPath, pathsSequence ); + + Log.Info( $"Dry run manifest written to {manifestPath}" ); + Log.Info( $"Dry run paths list written to {pathsOutputPath}" ); + } + + private static bool TryUploadArtifacts( IReadOnlyCollection<(string RepoPath, string AbsolutePath)> candidates, string remoteBase, List artifacts, string artifactLabel, bool skipUpload ) + { + if ( candidates.Count == 0 ) + { + Log.Info( $"No {artifactLabel} artifacts found to upload" ); + return true; + } + var uniqueUploads = new Dictionary( StringComparer.OrdinalIgnoreCase ); - foreach ( var absolutePath in filesToUpload ) + foreach ( var (repoPath, absolutePath) in candidates ) { + var repoPathNormalized = ToForwardSlash( repoPath ); + + if ( !File.Exists( absolutePath ) ) + { + Log.Error( $"Artifact not found on disk: {repoPathNormalized}" ); + return false; + } + var fileInfo = new FileInfo( absolutePath ); var sha256 = Utility.CalculateSha256( absolutePath ); - var repoRelativePath = Path.GetRelativePath( repositoryRoot, absolutePath ).Replace( '\\', '/' ); - var artifact = new ArtifactFileInfo { - Path = repoRelativePath, + Path = repoPathNormalized, Sha256 = sha256, Size = fileInfo.Length }; artifacts.Add( artifact ); - if ( !uniqueUploads.TryAdd( sha256, (absolutePath, artifact) ) ) { - Log.Info( $"Skipping upload for {repoRelativePath} (already uploaded hash {sha256})" ); + Log.Info( $"Skipping upload for {repoPathNormalized} (hash {sha256} already queued for upload)" ); } } + if ( uniqueUploads.Count == 0 ) + { + Log.Info( $"No unique {artifactLabel} artifacts to upload" ); + return true; + } + + if ( skipUpload ) + { + Log.Info( $"Dry run: skipping upload for {uniqueUploads.Count} unique {artifactLabel} artifacts" ); + return true; + } + var maxParallelUploads = Math.Max( 1, Math.Min( MAX_PARALLEL_UPLOADS, Environment.ProcessorCount ) ); - Log.Info( $"Uploading {uniqueUploads.Count} unique build artifacts (up to {maxParallelUploads} concurrent uploads)..." ); + Log.Info( $"Uploading {uniqueUploads.Count} unique {artifactLabel} artifacts (up to {maxParallelUploads} concurrent uploads)..." ); var failedUploads = new ConcurrentBag(); Parallel.ForEach( uniqueUploads, new ParallelOptions { MaxDegreeOfParallelism = maxParallelUploads }, kvp => @@ -440,67 +521,21 @@ internal class SyncPublicRepo( string name ) : Step( name ) var (absolutePath, artifact) = kvp.Value; if ( !UploadArtifactFile( absolutePath, artifact, remoteBase ) ) { - Log.Error( $"Failed to upload build artifact: {artifact.Path}" ); + Log.Error( $"Failed to upload {artifactLabel} artifact: {artifact.Path}" ); failedUploads.Add( artifact.Path ); } } ); if ( !failedUploads.IsEmpty ) { - Log.Error( $"Failed to upload {failedUploads.Count} build artifact(s)" ); + Log.Error( $"Failed to upload {failedUploads.Count} {artifactLabel} artifact(s)" ); return false; } - Log.Info( $"Uploaded {uniqueUploads.Count} unique build artifacts" ); + Log.Info( $"Uploaded {uniqueUploads.Count} unique {artifactLabel} artifacts" ); return true; } - private static bool RemovePathsFromRepo( IReadOnlyCollection pathsToRemove, string relativeRepoPath ) - { - if ( pathsToRemove.Count == 0 ) - { - return true; - } - - var tempFile = Path.GetTempFileName(); - - try - { - var normalizedPaths = new List( pathsToRemove.Count ); - foreach ( var path in pathsToRemove ) - { - normalizedPaths.Add( path.Replace( '\\', '/' ) ); - } - - File.WriteAllLines( tempFile, normalizedPaths ); - var args = $"filter-repo --force --paths-from-file \"{tempFile}\" --invert-paths"; - if ( !Utility.RunProcess( "git", args, relativeRepoPath ) ) - { - Log.Error( "Failed to remove LFS tracked files from repository" ); - return false; - } - - return true; - } - finally - { - if ( File.Exists( tempFile ) ) - { - File.Delete( tempFile ); - } - } - } - - private static string NormalizeRepoPath( string repoRelativePath ) - { - return repoRelativePath.Replace( '/', Path.DirectorySeparatorChar ); - } - - private static string BuildRcloneRemoteBase( string bucket, string accessKeyId, string secretAccessKey, string endpoint ) - { - return $":s3,bucket={bucket},provider=Cloudflare,access_key_id={accessKeyId},secret_access_key={secretAccessKey},endpoint='{endpoint}':"; - } - private static bool UploadArtifactFile( string localPath, ArtifactFileInfo artifact, string remoteBase ) { var remotePath = $"{remoteBase}/artifacts/{artifact.Sha256}"; @@ -519,10 +554,7 @@ internal class SyncPublicRepo( string name ) : Step( name ) Files = files }; - var manifestJson = JsonSerializer.Serialize( manifest, new JsonSerializerOptions - { - WriteIndented = true - } ); + var manifestJson = JsonSerializer.Serialize( manifest, new JsonSerializerOptions { WriteIndented = true } ); var manifestPath = Path.Combine( Path.GetTempPath(), $"{commitHash}.json" ); File.WriteAllText( manifestPath, manifestJson ); @@ -533,6 +565,7 @@ internal class SyncPublicRepo( string name ) : Step( name ) var remotePath = $"{remoteBase}/manifests/{commitHash}.json"; if ( !Utility.RunProcess( "rclone", $"copyto \"{manifestPath}\" \"{remotePath}\"", timeoutMs: 60000 ) ) { + Log.Error( "Failed to upload manifest file" ); return false; } } @@ -554,21 +587,20 @@ internal class SyncPublicRepo( string name ) : Step( name ) return string.IsNullOrEmpty( relativePath ) ? "." : relativePath; } - private static void TryDeleteDirectory( string path ) + private static string GetR2Base() { - if ( !Directory.Exists( path ) ) + var r2AccessKeyId = Environment.GetEnvironmentVariable( "SYNC_R2_ACCESS_KEY_ID" ); + var r2SecretAccessKey = Environment.GetEnvironmentVariable( "SYNC_R2_SECRET_ACCESS_KEY" ); + var r2Bucket = Environment.GetEnvironmentVariable( "SYNC_R2_BUCKET" ); + var r2Endpoint = Environment.GetEnvironmentVariable( "SYNC_R2_ENDPOINT" ); + + if ( string.IsNullOrEmpty( r2AccessKeyId ) || string.IsNullOrEmpty( r2SecretAccessKey ) || + string.IsNullOrEmpty( r2Bucket ) || string.IsNullOrEmpty( r2Endpoint ) ) { - return; + Log.Error( "R2 credentials not properly configured in environment variables" ); + return null; } - try - { - Log.Info( "Cleaning up temporary filtered repository..." ); - Directory.Delete( path, true ); - } - catch ( Exception ex ) - { - Log.Warning( $"Failed to clean up temporary directory: {ex.Message}" ); - } + return $":s3,bucket={r2Bucket},provider=Cloudflare,access_key_id={r2AccessKeyId},secret_access_key={r2SecretAccessKey},endpoint='{r2Endpoint}':"; } }