From ca8a9fb5a73e3d1a6742621fdc0155dd8984af0b Mon Sep 17 00:00:00 2001 From: jp9000 Date: Tue, 9 Dec 2014 13:33:16 -0800 Subject: [PATCH] libobs: Fix conversion shader D3D display bug Just for a quick background: D3D's fmod intrinsic is very imprecise. Naturally floating points aren't precise at all, and when the numbers you're dealing with become very large, it can often be off by 0.1 or more. However, apparently 0.1 isn't enough of an offset to ensure a proper value when using the fmod intrinsic and then flooring the value. 0.2 seems to fix the issue and make the image display properly. --- libobs/data/format_conversion.effect | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libobs/data/format_conversion.effect b/libobs/data/format_conversion.effect index 149279c45..8a77cec01 100644 --- a/libobs/data/format_conversion.effect +++ b/libobs/data/format_conversion.effect @@ -59,7 +59,7 @@ VertInOut VSDefault(VertInOut vert_in) } /* used to prevent internal GPU precision issues width fmod in particular */ -#define PRECISION_OFFSET 0.1 +#define PRECISION_OFFSET 0.2 float4 PSNV12(VertInOut vert_in) : TARGET {