From 5d3fbc15aeb68a2f5bbee1fb9655e9d43d65f063 Mon Sep 17 00:00:00 2001 From: Theodore Dubois Date: Sun, 16 Feb 2020 12:28:50 -0800 Subject: [PATCH] obs-vth264: Manually mark priority bits for VideoToolbox frames Reland of 30d29618, except actually tested this time. The VideoToolbox encoder gives I-frames and P-frames a priority of 1, but the RTMP output code expects I-frames to have priority 3 and P-frames to have priority 2. 30d29618 changed the priority of all frames that aren't I-frames, but that included B-frames as well as P-frames. B-frames are given a priority of 0 by VideoToolbox and changing that priority causes artifacts for reasons I don't understand. So ignore the B-frames by ignoring slice packets with priority 0. --- plugins/mac-vth264/encoder.c | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/plugins/mac-vth264/encoder.c b/plugins/mac-vth264/encoder.c index b91b6d199..6c062370a 100644 --- a/plugins/mac-vth264/encoder.c +++ b/plugins/mac-vth264/encoder.c @@ -1,5 +1,6 @@ #include #include +#include #include #include @@ -671,6 +672,8 @@ static bool is_sample_keyframe(CMSampleBufferRef buffer) static bool parse_sample(struct vt_h264_encoder *enc, CMSampleBufferRef buffer, struct encoder_packet *packet, CMTime off) { + int type; + CMTime pts = CMSampleBufferGetPresentationTimeStamp(buffer); CMTime dts = CMSampleBufferGetDecodeTimeStamp(buffer); @@ -703,6 +706,37 @@ static bool parse_sample(struct vt_h264_encoder *enc, CMSampleBufferRef buffer, packet->size = enc->packet_data.num; packet->keyframe = keyframe; + // VideoToolbox produces packets with priority lower than the RTMP code + // expects, which causes it to be unable to recover from frame drops. + // Fix this by manually adjusting the priority. + uint8_t *start = enc->packet_data.array; + uint8_t *end = start + enc->packet_data.num; + + start = (uint8_t *)obs_avc_find_startcode(start, end); + while (true) { + while (start < end && !*(start++)) + ; + + if (start == end) + break; + + type = start[0] & 0x1F; + if (type == OBS_NAL_SLICE_IDR || type == OBS_NAL_SLICE) { + uint8_t prev_type = (start[0] >> 5) & 0x3; + start[0] &= ~(3 << 5); + + if (type == OBS_NAL_SLICE_IDR) + start[0] |= OBS_NAL_PRIORITY_HIGHEST << 5; + else if (type == OBS_NAL_SLICE && + prev_type != OBS_NAL_PRIORITY_DISPOSABLE) + start[0] |= OBS_NAL_PRIORITY_HIGH << 5; + else + start[0] |= prev_type << 5; + } + + start = (uint8_t *)obs_avc_find_startcode(start, end); + } + CFRelease(buffer); return true;