Vulkan Raytracing Support (#2409)

* Re-enable CreateBLAS in RenderDeviceVulkan

* Update SPVRemapper to suppot raytracing opcode remapping

* Null initialize BLAS on RenderMesh

* Clean proper to generate BLAS on mesh data loading

* SceneRaytracingSystem stub

* Glue SceneRaytracing

* Remove pipelines from SceneRaytracing for now, just do TLAS, tie it to SceneRaytracingSystem, updates only once per frame just like how we want in a clean way
https://files.facepunch.com/sampavlovic/1b0611b1/ngfx-ui_Ck3ckzQQFT.png

* Send Descriptor Set of Raytracing World to RenderAttribute

* RTAO example using RayQuery

* RayTracingShader API stub

* Set SM6.0 and dxlevel 120 as default

* Instead of making raytracing opt-in, add -nogpuraytracing to force disable it

* Add IRenderDevice::IsRayTracingSupported() to easily query support for it

* Fix IsRayTracingSupported()

* RTAO Adjustments

* Allow Rayquery through AS on Compute and Pixel shaders even if not optimal, avoids it crashing when acessing it on compute

* Strip CRaytraceSceneWorld to just generating TLAS, dont need anything else for now and we need a better way to handle UGC than what's there

* Bindless::RaytracingAccelerationStructure()

* Simplify interface too

* Stub for UpdateSkinnedForRaytracing

* Dotnet format and fix documentation, make RTAO run at correct stage

* Only support raytracing in tools mode for now

* Move raytracing stuff to Raytracing.hlsl

* Stub RTX shader

* Internal Raytracingshader and remove useless stuff exposed from it

* VfxProgramHasRenderShader should check for VFX_PROGRAM_RTX too, that's the source from everything else failing

* Add arbitrary entry point support to shaders, needed as RTX shaders use [shader("XXX")] instead of MainXXX directly

* RenderTools::TraceRays API, preliminary implementation, RTAO uses RaytracingShader

* Make RT support internal, allow RT on game

* Remove RaytracedAmbientOcclusion, will be on scenestaging when ready

* Update engine/Sandbox.Engine/Systems/Render/RayTracingShader.cs

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Fix rebase

* Update shaders

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

engine/Definitions/common/Render/renderdevice.def

@@ -69,6 +69,8 @@ native accessor g_pRenderDevice
 
 	bool IsTextureRenderTarget( ITexture texture );
 
+	bool IsRayTracingSupported();
+
 	inline RenderBufferHandle_t CreateGPUBuffer( RenderBufferType_t nType, BufferDesc_t desc, RenderBufferFlags_t usage, string pDebugName )
 	{
 		desc.m_pDebugName = pDebugName;

engine/Definitions/common/SceneSystem/IRayTraceSceneWorld.def

@@ -0,0 +1,19 @@
+#include "scenesystem/iscenesystem.h"
+
+native class IRayTraceSceneWorld as NativeEngine.IRayTraceSceneWorld
+{
+    void BeginBuild();
+    void AddSceneWorldToBuild( ISceneWorld pWorld, IRenderContext pRenderContext );
+	bool EndBuild( IRenderContext pRenderContext, CRenderAttributes attrs );
+
+    inline bool BuildTLASForWorld( ISceneWorld pWorld, CRenderAttributes attrs )
+    {
+        CRenderContextPtr pRenderContext( g_pRenderDevice, "sceneinfo_gpu" );
+
+        this->BeginBuild();
+        this->AddSceneWorldToBuild( pWorld, pRenderContext );
+        this->EndBuild( pRenderContext, attrs );
+
+        return true;
+    } 
+}

engine/Definitions/common/SceneSystem/g_pSceneSystem.def

@@ -160,4 +160,8 @@ native accessor g_pSceneSystem as NativeEngine.CSceneSystem
 	void RenderTiledLightCulling( IRenderContext pCtx, ISceneView pView, RenderViewport viewport );
 
 	void BindTransformSlot( IRenderContext pCtx, int nVBSlot, int nTransformSlotIndex );
+
+	// Ray Tracing
+	IRayTraceSceneWorld CreateRayTraceWorld( string pWorldDebugName, int nMaxRayTypes );
+	void DestroyRayTraceWorld( IRayTraceSceneWorld pRayTraceSceneWorld );
 }

engine/Definitions/engine/Render/RenderTools.def

@@ -17,6 +17,8 @@ native static class RenderTools as NativeEngine.RenderTools
 	//
 	void Compute( IRenderContext renderContext, CRenderAttributes attributes, IMaterialMode pMode, int tx, int ty, int tz );
 	void ComputeIndirect( IRenderContext renderContext, CRenderAttributes attributes, IMaterialMode pMode, RenderBufferHandle_t hIndirectBuffer, uint nIndirectBufferOffset );
+	void TraceRays( IRenderContext renderContext, CRenderAttributes attributes, IMaterialMode pMode, uint tx, uint ty, uint tz );
+	void TraceRaysIndirect( IRenderContext renderContext, CRenderAttributes attributes, IMaterialMode pMode, RenderBufferHandle_t hIndirectBuffer, uint nIndirectBufferOffset );
 	void SetDynamicConstantBufferData( CRenderAttributes attributes, StringToken nTokenID, IRenderContext renderContext, void* data, int dataSize );
 
 	void CopyTexture( IRenderContext renderContext, ITexture sourceTexture, ITexture destTexture, Rect_t pSrcRect, int nDestX, int nDestY, uint nSrcMipSlice, uint nSrcArraySlice, uint nDstMipSlice, uint nDstArraySlice );

engine/Sandbox.Engine/Scene/GameObjectSystems/SceneRaytracingSystem.cs

@@ -0,0 +1,57 @@
+using NativeEngine;
+
+namespace Sandbox;
+
+internal class SceneRaytracingSystem : GameObjectSystem<SceneRaytracingSystem>
+{
+	// This could all be done on C# side, all below does is to update the TLAS, todo
+	internal IRayTraceSceneWorld native;
+
+	bool Supported => g_pRenderDevice.IsRayTracingSupported();
+
+	public SceneRaytracingSystem( Sandbox.Scene scene ) : base( scene )
+	{
+		if ( !Supported )
+			return;
+		// Update TLAS after bones are updated
+		Listen( Stage.UpdateBones, Int32.MaxValue, UpdateSkinnedForRaytracing, "UpdateSkinnedForRaytracing" );
+		Listen( Stage.FinishUpdate, Int32.MaxValue, UpdateRaytracing, "UpdateRaytracing" );
+
+		var RAY_TYPE_COUNT = 2;
+		native = CSceneSystem.CreateRayTraceWorld( Scene.Name, RAY_TYPE_COUNT );
+	}
+
+	~SceneRaytracingSystem()
+	{
+		if ( !Supported )
+			return;
+
+		CSceneSystem.DestroyRayTraceWorld( native );
+	}
+
+
+	void UpdateSkinnedForRaytracing()
+	{
+		using var _ = PerformanceStats.Timings.Render.Scope();
+
+		var allSkinnedRenderers = Scene.GetAllComponents<SkinnedModelRenderer>()
+										.ToArray();
+
+		foreach ( var renderer in allSkinnedRenderers )
+		{
+			if ( !renderer.IsValid() )
+				continue;
+
+		}
+	}
+
+	/// <summary>
+	/// Top-level acceleration structure needs to be updated every frame
+	/// </summary>
+	internal void UpdateRaytracing()
+	{
+		using var _ = PerformanceStats.Timings.Render.Scope();
+
+		native.BuildTLASForWorld( Scene.SceneWorld, Scene.RenderAttributes.Get() );
+	}
+}

engine/Sandbox.Engine/Systems/Render/CommandList/CommandList.cs

@@ -830,6 +830,28 @@ public sealed unsafe partial class CommandList
 		AddEntry( &Execute, new Entry { Object1 = compute, Object2 = indirectBuffer, Data1 = new Vector4( indirectElementOffset, 0, 0, 0 ) } );
 	}
 
+	/// <inheritdoc cref="RayTracingShader.DispatchRaysWithAttributes(RenderAttributes, int, int, int)"/>
+	internal void DispatchRays( RayTracingShader raytracing, int threadsX, int threadsY, int threadsZ )
+	{
+		static void Execute( ref Entry entry, CommandList commandList )
+		{
+			((RayTracingShader)entry.Object1).DispatchRaysWithAttributes( Graphics.Attributes, (int)entry.Data1.x, (int)entry.Data1.y, (int)entry.Data1.z );
+		}
+
+		AddEntry( &Execute, new Entry { Object1 = raytracing, Data1 = new Vector4( threadsX, threadsY, threadsZ, 0 ) } );
+	}
+
+	/// <inheritdoc cref="RayTracingShader.DispatchRaysIndirect(GpuBuffer, uint)"/>
+	internal void DispatchRaysIndirect( RayTracingShader raytracing, GpuBuffer indirectBuffer, uint indirectElementOffset = 0 )
+	{
+		static void Execute( ref Entry entry, CommandList commandList )
+		{
+			((RayTracingShader)entry.Object1).DispatchRaysIndirectWithAttributes( Graphics.Attributes, (GpuBuffer)entry.Object2, (uint)entry.Data1.x );
+		}
+
+		AddEntry( &Execute, new Entry { Object1 = raytracing, Object2 = indirectBuffer, Data1 = new Vector4( indirectElementOffset, 0, 0, 0 ) } );
+	}
+
 	/// <summary>
 	/// A handle to the viewport size
 	/// </summary>
@@ -851,6 +873,22 @@ public sealed unsafe partial class CommandList
 		AddEntry( &Execute, new Entry { Object1 = compute, Object5 = dimension.Name } );
 	}
 
+	/// <summary>
+	/// Dispatch a ray tracing shader
+	/// </summary>
+	internal void DispatchRays( RayTracingShader raytracing, RenderTargetHandle.SizeHandle dimension )
+	{
+		static void Execute( ref Entry entry, CommandList commandList )
+		{
+			var xyz = commandList.GetDimension( (string)entry.Object5 );
+			if ( !xyz.HasValue ) return;
+
+			((RayTracingShader)entry.Object1).DispatchRaysWithAttributes( Graphics.Attributes, xyz.Value.x, xyz.Value.y, xyz.Value.z );
+		}
+
+		AddEntry( &Execute, new Entry { Object1 = raytracing, Object5 = dimension.Name } );
+	}
+
 	/// <summary>
 	/// Called during rendering, convert RenderTargetHandle.SizeHandle to a dimension
 	/// </summary>

engine/Sandbox.Engine/Systems/Render/RayTracingAccelerationStructure.cs

@@ -0,0 +1,66 @@
+using NativeEngine;
+
+namespace Sandbox;
+
+/// <summary>
+/// Represents a ray tracing acceleration structure that contains geometry for efficient ray intersection testing.
+/// This is used to organize scene geometry in a hierarchical structure optimized for ray tracing performance.
+/// </summary>
+public class RayTracingAccelerationStructure
+{
+	internal object native;
+
+	/// <summary>
+	/// Gets whether this acceleration structure is valid and can be used for ray tracing.
+	/// </summary>
+	public bool IsValid() => native != null;
+
+	/// <summary>
+	/// Create a ray tracing acceleration structure from native engine data.
+	/// </summary>
+	internal RayTracingAccelerationStructure( object nativeAccelerationStructure )
+	{
+		native = nativeAccelerationStructure;
+	}
+
+	/// <summary>
+	/// Create a ray tracing acceleration structure from scene geometry.
+	/// </summary>
+	/// <param name="geometryData">The geometry data to build the acceleration structure from.</param>
+	/// <returns>A new acceleration structure, or null if creation failed.</returns>
+	public static RayTracingAccelerationStructure Create( object geometryData )
+	{
+		// This would typically interface with the native engine to build the acceleration structure
+		// For now, this is a placeholder implementation
+		if ( geometryData == null )
+			return null;
+
+		// In a real implementation, this would call into the native engine
+		// to build a DXR acceleration structure from the provided geometry
+		var nativeAS = geometryData; // Placeholder
+
+		return new RayTracingAccelerationStructure( nativeAS );
+	}
+
+	/// <summary>
+	/// Updates the acceleration structure with new geometry data.
+	/// This is more efficient than rebuilding from scratch for dynamic geometry.
+	/// </summary>
+	/// <param name="geometryData">The updated geometry data.</param>
+	public void Update( object geometryData )
+	{
+		if ( !IsValid() )
+			throw new InvalidOperationException( "Cannot update invalid acceleration structure" );
+
+		// This would call into the native engine to update the acceleration structure
+		// with new geometry positions while preserving the hierarchical structure
+	}
+
+	/// <summary>
+	/// Releases the native resources associated with this acceleration structure.
+	/// </summary>
+	public void Dispose()
+	{
+		native = null;
+	}
+}

engine/Sandbox.Engine/Systems/Render/RayTracingShader.cs

@@ -0,0 +1,106 @@
+using NativeEngine;
+
+namespace Sandbox;
+
+/// <summary>
+/// A ray tracing shader,
+/// enabling advanced rendering techniques like real-time ray tracing for reflections, 
+/// global illumination, and shadows.
+/// </summary>
+/// <seealso cref="GpuBuffer{T}"/>
+/// <seealso cref="ComputeShader"/>
+internal class RayTracingShader
+{
+	/// <summary>
+	/// Attributes that are passed to the ray tracing shader on dispatch.
+	/// </summary>
+	public RenderAttributes Attributes { get; } = new RenderAttributes();
+
+	private Material RayTracingMaterial;
+
+	/// <summary>
+	/// Create a ray tracing shader from the specified path.
+	/// </summary>
+	public RayTracingShader( string path )
+	{
+		var material = Material.FromShader( path );
+		Assert.NotNull( material, $"Failed to load ray tracing shader material from path: {path}" );
+		RayTracingMaterial = material;
+	}
+
+	/// <summary>
+	/// Dispatches the ray tracing shader using explicit thread counts.
+	/// </summary>
+	/// <remarks>
+	/// The specified thread counts represent the dispatch dimensions for the ray generation shader.
+	/// <para>
+	/// When called outside a graphics context, the dispatch runs immediately.  
+	/// When called inside a graphics context, the dispatch runs async.
+	/// </para>
+	/// </remarks>
+	/// <param name="attributes">Render attributes to use for this dispatch.</param>
+	/// <param name="threadsX">The number of threads to dispatch in the X dimension.</param>
+	/// <param name="threadsY">The number of threads to dispatch in the Y dimension.</param>
+	/// <param name="threadsZ">The number of threads to dispatch in the Z dimension.</param>
+	public void DispatchRaysWithAttributes( RenderAttributes attributes, int threadsX = 1, int threadsY = 1, int threadsZ = 1 )
+	{
+		if ( threadsX < 1 ) throw new ArgumentException( $"Cannot be less than 1", nameof( threadsX ) );
+		if ( threadsY < 1 ) throw new ArgumentException( $"Cannot be less than 1", nameof( threadsY ) );
+		if ( threadsZ < 1 ) throw new ArgumentException( $"Cannot be less than 1", nameof( threadsZ ) );
+
+		// Dispatch ray tracing using RenderTools.TraceRays
+		var mode = RayTracingMaterial.native.GetMode();
+		RenderTools.TraceRays( Graphics.Context, attributes.Get(), mode, (uint)threadsX, (uint)threadsY, (uint)threadsZ );
+	}
+
+	/// <summary>
+	/// Dispatches the ray tracing shader using the default attributes.
+	/// </summary>
+	public void DispatchRays( int threadsX = 1, int threadsY = 1, int threadsZ = 1 )
+	{
+		DispatchRaysWithAttributes( Attributes, threadsX, threadsY, threadsZ );
+	}
+
+	/// <summary>
+	/// Dispatches the ray tracing shader by reading dispatch arguments from an indirect buffer.
+	/// </summary>
+	/// <remarks>
+	/// <para>
+	/// <paramref name="indirectBuffer"/> must be created with <see cref="GpuBuffer.UsageFlags.IndirectDrawArguments"/>  
+	/// and have an element size of 12 bytes (3 uint32 values for X, Y, Z dimensions).
+	/// </para>
+	/// <para>
+	/// <paramref name="indirectElementOffset"/> is an element index into <paramref name="indirectBuffer"/>, not a byte offset.
+	/// </para>
+	/// <para>
+	/// When called outside a graphics context, the dispatch runs immediately.  
+	/// When called inside a graphics context, the dispatch runs async.
+	/// </para>
+	/// </remarks>
+	/// <param name="indirectBuffer">The GPU buffer containing one or more dispatch argument entries.</param>
+	/// <param name="indirectElementOffset">The index of the dispatch arguments element to use (each element = 12 bytes).</param>
+	public void DispatchRaysIndirect( GpuBuffer indirectBuffer, uint indirectElementOffset = 0 )
+	{
+		DispatchRaysIndirectWithAttributes( Attributes, indirectBuffer, indirectElementOffset );
+	}
+
+	/// <inheritdoc cref="DispatchRaysIndirect"/>
+	public void DispatchRaysIndirectWithAttributes( RenderAttributes attributes, GpuBuffer indirectBuffer, uint indirectElementOffset = 0 )
+	{
+		if ( !indirectBuffer.IsValid() )
+			throw new ArgumentException( $"Invalid buffer", nameof( indirectBuffer ) );
+
+		if ( indirectBuffer.ElementSize != 12 )
+			throw new ArgumentException( $"Buffer element size must be 12 bytes", nameof( indirectBuffer ) );
+
+		if ( indirectElementOffset >= indirectBuffer.ElementCount )
+			throw new ArgumentOutOfRangeException( nameof( indirectElementOffset ), "Indirect element offset exceeds buffer bounds" );
+
+		if ( !indirectBuffer.Usage.Contains( GpuBuffer.UsageFlags.IndirectDrawArguments ) )
+			throw new ArgumentException( $"Buffer must have the required usage flag '{GpuBuffer.UsageFlags.IndirectDrawArguments}'", nameof( indirectBuffer ) );
+
+		// Use RenderTools.TraceRaysIndirect when it becomes available, for now use the material mode directly
+		var mode = RayTracingMaterial.native.GetMode();
+		RenderTools.TraceRaysIndirect( Graphics.Context, attributes.Get(), mode, indirectBuffer.native, indirectElementOffset * 12 );
+	}
+}

game/addons/base/Assets/shaders/common/classes/Raytracing.hlsl

@@ -0,0 +1,20 @@
+#ifndef RAYTRACING_HLSL
+#define RAYTRACING_HLSL
+
+ExternalDescriptorSet RaytracingDescriptorSet Slot 0;
+RaytracingAccelerationStructure _accelStruct EXTERNAL_DESC_SET(t, RaytracingDescriptorSet, 0);
+
+class Raytracing
+{
+    static RaytracingAccelerationStructure GetAccelerationStructure() { return _accelStruct; }
+    static float3 GetAlbedoForInstance(int instanceID) { return 1.0f; /* Placeholder white albedo */ }
+
+    struct Result
+    {
+        bool Hit;
+        float3 HitPosition;
+        int HitInstanceID;
+    };
+};
+
+#endif

game/addons/base/Assets/shaders/common/classes/_classes.hlsl

@@ -13,4 +13,4 @@
 #include "common/classes/ScreenSpaceTrace.hlsl"
 #include "common/classes/AmbientLight.hlsl"
 #include "common/classes/EnvMap.hlsl"
-#include "common/classes/ToolsVis.hlsl"
+#include "common/classes/ToolsVis.hlsl"