From edfc2bed05218b9d6ee41f810f0b140e0497409d Mon Sep 17 00:00:00 2001
From: jp9000 <obs.jim@gmail.com>
Date: Mon, 14 Nov 2016 17:31:22 -0800
Subject: [PATCH] obs-qsv11: Manually mark priority bits for QSV frames

QSV frame priority bits are not being marked correctly; their priorities
are always marked either low or disposable, whereas I-frames should be
marked as highest (3), P-frames should be marked as high (2), and other
frames should be marked as either low or disposable.  This helps ensure
that the correct frames are dropped when frames need to be dropped.
---
 plugins/obs-qsv11/obs-qsv11.c | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/plugins/obs-qsv11/obs-qsv11.c b/plugins/obs-qsv11/obs-qsv11.c
index 42002f2ac..b57f8a18a 100644
--- a/plugins/obs-qsv11/obs-qsv11.c
+++ b/plugins/obs-qsv11/obs-qsv11.c
@@ -59,6 +59,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <util/darray.h>
 #include <util/platform.h>
 #include <obs-module.h>
+#include <obs-avc.h>
 
 #ifndef _STDINT_H_INCLUDED
 #define _STDINT_H_INCLUDED
@@ -579,6 +580,9 @@ static void obs_qsv_video_info(void *data, struct video_scale_info *info)
 
 static void parse_packet(struct obs_qsv *obsqsv, struct encoder_packet *packet, mfxBitstream *pBS, uint32_t fps_num, bool *received_packet)
 {
+	uint8_t *start, *end;
+	int type;
+
 	if (pBS == NULL || pBS->DataLength == 0) {
 		*received_packet = false;
 		return;
@@ -595,6 +599,36 @@ static void parse_packet(struct obs_qsv *obsqsv, struct encoder_packet *packet,
 	packet->keyframe = (pBS->FrameType &
 			(MFX_FRAMETYPE_I | MFX_FRAMETYPE_REF));
 
+	/* ------------------------------------ */
+
+	start = obsqsv->packet_data.array;
+	end = start + obsqsv->packet_data.num;
+
+	start = (uint8_t*)obs_avc_find_startcode(start, end);
+	while (true) {
+		while (start < end && !*(start++));
+
+		if (start == end)
+			break;
+
+		type = start[0] & 0x1F;
+		if (type == OBS_NAL_SLICE_IDR || type == OBS_NAL_SLICE) {
+			uint8_t prev_type = (start[0] >> 5) & 0x3;
+			start[0] &= ~(3 << 5);
+
+			if (pBS->FrameType & MFX_FRAMETYPE_I)
+				start[0] |= OBS_NAL_PRIORITY_HIGHEST << 5;
+			else if (pBS->FrameType & MFX_FRAMETYPE_P)
+				start[0] |= OBS_NAL_PRIORITY_HIGH << 5;
+			else
+				start[0] |= prev_type << 5;
+		}
+
+		start = (uint8_t*)obs_avc_find_startcode(start, end);
+	}
+
+	/* ------------------------------------ */
+
 	//bool iFrame = pBS->FrameType & MFX_FRAMETYPE_I;
 	//bool bFrame = pBS->FrameType & MFX_FRAMETYPE_B;
 	bool pFrame = pBS->FrameType & MFX_FRAMETYPE_P;