From a7e81d6f3f5d43149dbf90291d543c1442601c9b Mon Sep 17 00:00:00 2001 From: John Bradley Date: Thu, 19 Mar 2015 10:45:58 -0500 Subject: [PATCH 01/23] deps-libff: (unsigned char *) -> (uint8_t *) cast fix --- deps/libff/libff/ff-packet-queue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/libff/libff/ff-packet-queue.c b/deps/libff/libff/ff-packet-queue.c index e1f6350fc..ca3de20cc 100644 --- a/deps/libff/libff/ff-packet-queue.c +++ b/deps/libff/libff/ff-packet-queue.c @@ -27,7 +27,7 @@ bool packet_queue_init(struct ff_packet_queue *q) goto fail1; av_init_packet(&q->flush_packet); - q->flush_packet.data = (unsigned char *) "FLUSH"; + q->flush_packet.data = (uint8_t *)"FLUSH"; return true; From b91a98ed44fc57397fa876e3e329085ac61bc1db Mon Sep 17 00:00:00 2001 From: John Bradley Date: Thu, 19 Mar 2015 11:00:42 -0500 Subject: [PATCH 02/23] deps-libff: Refactor AVPacket into ff_packet This also replaces AVPacketList with ff_packet_list. --- deps/libff/libff/ff-audio-decoder.c | 33 ++++++++++++++++------------- deps/libff/libff/ff-decoder.c | 4 ++-- deps/libff/libff/ff-decoder.h | 2 +- deps/libff/libff/ff-demuxer.c | 6 +++--- deps/libff/libff/ff-packet-queue.c | 32 +++++++++++++++------------- deps/libff/libff/ff-packet-queue.h | 20 ++++++++++++----- deps/libff/libff/ff-video-decoder.c | 8 +++---- 7 files changed, 60 insertions(+), 45 deletions(-) diff --git a/deps/libff/libff/ff-audio-decoder.c b/deps/libff/libff/ff-audio-decoder.c index b3ebe28e1..b31a6633d 100644 --- a/deps/libff/libff/ff-audio-decoder.c +++ b/deps/libff/libff/ff-audio-decoder.c @@ -28,29 +28,30 @@ #include -static inline void shrink_packet(AVPacket *packet, int packet_length) +static inline void shrink_packet(struct ff_packet *packet, int packet_length) { - if (packet_length <= packet->size) { - int remaining = packet->size - packet_length; + if (packet_length <= packet->base.size) { + int remaining = packet->base.size - packet_length; - memmove(packet->data, &packet->data[packet_length], remaining); - av_shrink_packet(packet, remaining); + memmove(packet->base.data, &packet->base.data[packet_length], + remaining); + av_shrink_packet(&packet->base, remaining); } } static int decode_frame(struct ff_decoder *decoder, - AVPacket *packet, AVFrame *frame, bool *frame_complete) + struct ff_packet *packet, AVFrame *frame, bool *frame_complete) { int packet_length; int ret; while (true) { - while (packet->size > 0) { + while (packet->base.size > 0) { int complete; packet_length = avcodec_decode_audio4(decoder->codec, frame, &complete, - packet); + &packet->base); if (packet_length < 0) break; @@ -66,15 +67,16 @@ static int decode_frame(struct ff_decoder *decoder, av_get_bytes_per_sample(frame->format); } - if (packet->data != NULL) - av_packet_unref(packet); + if (packet->base.data != NULL) + av_packet_unref(&packet->base); ret = packet_queue_get(&decoder->packet_queue, packet, 1); if (ret == FF_PACKET_FAIL) { return -1; } - if (packet->data == decoder->packet_queue.flush_packet.data) { + if (packet->base.data == + decoder->packet_queue.flush_packet.base.data) { avcodec_flush_buffers(decoder->codec); // we were flushed, so try to get another packet @@ -126,13 +128,14 @@ void *ff_audio_decoder_thread(void *opaque_audio_decoder) { struct ff_decoder *decoder = opaque_audio_decoder; - AVPacket packet = {0}; + struct ff_packet packet = {0}; bool frame_complete; AVFrame *frame = av_frame_alloc(); while (!decoder->abort) { - if (decode_frame(decoder, &packet, frame, &frame_complete) < 0) { - av_free_packet(&packet); + if (decode_frame(decoder, &packet, frame, &frame_complete) + < 0) { + av_free_packet(&packet.base); continue; } @@ -148,7 +151,7 @@ void *ff_audio_decoder_thread(void *opaque_audio_decoder) av_frame_unref(frame); } - av_free_packet(&packet); + av_free_packet(&packet.base); } av_frame_free(&frame); diff --git a/deps/libff/libff/ff-decoder.c b/deps/libff/libff/ff-decoder.c index 2d1ecf7f7..a9fd5fe62 100644 --- a/deps/libff/libff/ff-decoder.c +++ b/deps/libff/libff/ff-decoder.c @@ -245,9 +245,9 @@ bool ff_decoder_full(struct ff_decoder *decoder) return (decoder->packet_queue.total_size > decoder->packet_queue_size); } -bool ff_decoder_accept(struct ff_decoder *decoder, AVPacket *packet) +bool ff_decoder_accept(struct ff_decoder *decoder, struct ff_packet *packet) { - if (decoder && packet->stream_index == decoder->stream->index) { + if (decoder && packet->base.stream_index == decoder->stream->index) { packet_queue_put(&decoder->packet_queue, packet); return true; } diff --git a/deps/libff/libff/ff-decoder.h b/deps/libff/libff/ff-decoder.h index 8e2c882ff..4f2a75176 100644 --- a/deps/libff/libff/ff-decoder.h +++ b/deps/libff/libff/ff-decoder.h @@ -59,7 +59,7 @@ bool ff_decoder_start(struct ff_decoder *decoder); void ff_decoder_free(struct ff_decoder *decoder); bool ff_decoder_full(struct ff_decoder *decoder); -bool ff_decoder_accept(struct ff_decoder *decoder, AVPacket *packet); +bool ff_decoder_accept(struct ff_decoder *decoder, struct ff_packet *packet); double ff_decoder_clock(void *opaque); diff --git a/deps/libff/libff/ff-demuxer.c b/deps/libff/libff/ff-demuxer.c index 49b28103d..55eb7d286 100644 --- a/deps/libff/libff/ff-demuxer.c +++ b/deps/libff/libff/ff-demuxer.c @@ -448,7 +448,7 @@ static void *demux_thread(void *opaque) struct ff_demuxer *demuxer = (struct ff_demuxer *) opaque; int result; - AVPacket packet; + struct ff_packet packet = {0}; if (!open_input(demuxer, &demuxer->format_context)) goto fail; @@ -469,7 +469,7 @@ static void *demux_thread(void *opaque) continue; } - result = av_read_frame(demuxer->format_context, &packet); + result = av_read_frame(demuxer->format_context, &packet.base); if (result < 0) { bool eof = false; if (result == AVERROR_EOF) { @@ -506,7 +506,7 @@ static void *demux_thread(void *opaque) else if (ff_decoder_accept(demuxer->audio_decoder, &packet)) continue; else - av_free_packet(&packet); + av_free_packet(&packet.base); } if (demuxer->audio_decoder != NULL) demuxer->audio_decoder->eof = true; diff --git a/deps/libff/libff/ff-packet-queue.c b/deps/libff/libff/ff-packet-queue.c index ca3de20cc..4bf63d379 100644 --- a/deps/libff/libff/ff-packet-queue.c +++ b/deps/libff/libff/ff-packet-queue.c @@ -26,8 +26,8 @@ bool packet_queue_init(struct ff_packet_queue *q) if (pthread_cond_init(&q->cond, NULL) != 0) goto fail1; - av_init_packet(&q->flush_packet); - q->flush_packet.data = (uint8_t *)"FLUSH"; + av_init_packet(&q->flush_packet.base); + q->flush_packet.base.data = (uint8_t *)"FLUSH"; return true; @@ -53,22 +53,23 @@ void packet_queue_free(struct ff_packet_queue *q) pthread_mutex_destroy(&q->mutex); pthread_cond_destroy(&q->cond); - av_free_packet(&q->flush_packet); + av_free_packet(&q->flush_packet.base); } -int packet_queue_put(struct ff_packet_queue *q, AVPacket *packet) +int packet_queue_put(struct ff_packet_queue *q, struct ff_packet *packet) { - AVPacketList *new_packet; + struct ff_packet_list *new_packet; - if (packet != &q->flush_packet && av_dup_packet(packet) < 0) + if (packet != &q->flush_packet + && av_dup_packet(&packet->base) < 0) return FF_PACKET_FAIL; - new_packet = av_malloc(sizeof(AVPacketList)); + new_packet = av_malloc(sizeof(struct ff_packet_list)); if (new_packet == NULL) return FF_PACKET_FAIL; - new_packet->pkt = *packet; + new_packet->packet = *packet; new_packet->next = NULL; pthread_mutex_lock(&q->mutex); @@ -81,7 +82,7 @@ int packet_queue_put(struct ff_packet_queue *q, AVPacket *packet) q->last_packet = new_packet; q->count++; - q->total_size += new_packet->pkt.size; + q->total_size += new_packet->packet.base.size; pthread_cond_signal(&q->cond); pthread_mutex_unlock(&q->mutex); @@ -94,9 +95,10 @@ int packet_queue_put_flush_packet(struct ff_packet_queue *q) return packet_queue_put(q, &q->flush_packet); } -int packet_queue_get(struct ff_packet_queue *q, AVPacket *packet, bool block) +int packet_queue_get(struct ff_packet_queue *q, struct ff_packet *packet, + bool block) { - AVPacketList *potential_packet; + struct ff_packet_list *potential_packet; int return_status; pthread_mutex_lock(&q->mutex); @@ -111,8 +113,8 @@ int packet_queue_get(struct ff_packet_queue *q, AVPacket *packet, bool block) q->last_packet = NULL; q->count--; - q->total_size -= potential_packet->pkt.size; - *packet = potential_packet->pkt; + q->total_size -= potential_packet->packet.base.size; + *packet = potential_packet->packet; av_free(potential_packet); return_status = FF_PACKET_SUCCESS; break; @@ -137,14 +139,14 @@ int packet_queue_get(struct ff_packet_queue *q, AVPacket *packet, bool block) void packet_queue_flush(struct ff_packet_queue *q) { - AVPacketList *packet; + struct ff_packet_list *packet; pthread_mutex_lock(&q->mutex); for (packet = q->first_packet; packet != NULL; packet = q->first_packet) { q->first_packet = packet->next; - av_free_packet(&packet->pkt); + av_free_packet(&packet->packet.base); av_freep(&packet); } diff --git a/deps/libff/libff/ff-packet-queue.h b/deps/libff/libff/ff-packet-queue.h index d91e829be..26e00dcf6 100644 --- a/deps/libff/libff/ff-packet-queue.h +++ b/deps/libff/libff/ff-packet-queue.h @@ -24,12 +24,21 @@ #define FF_PACKET_EMPTY 0 #define FF_PACKET_SUCCESS 1 +struct ff_packet { + AVPacket base; +}; + +struct ff_packet_list { + struct ff_packet packet; + struct ff_packet_list *next; +}; + struct ff_packet_queue { - AVPacketList *first_packet; - AVPacketList *last_packet; + struct ff_packet_list *first_packet; + struct ff_packet_list *last_packet; pthread_mutex_t mutex; pthread_cond_t cond; - AVPacket flush_packet; + struct ff_packet flush_packet; int count; unsigned int total_size; bool abort; @@ -40,8 +49,9 @@ typedef struct ff_packet_queue ff_packet_queue_t; bool packet_queue_init(struct ff_packet_queue *q); void packet_queue_abort(struct ff_packet_queue *q); void packet_queue_free(struct ff_packet_queue *q); -int packet_queue_put(struct ff_packet_queue *q, AVPacket *packet); +int packet_queue_put(struct ff_packet_queue *q, struct ff_packet *packet); int packet_queue_put_flush_packet(struct ff_packet_queue *q); -int packet_queue_get(struct ff_packet_queue *q, AVPacket *packet, bool block); +int packet_queue_get(struct ff_packet_queue *q, struct ff_packet *packet, + bool block); void packet_queue_flush(struct ff_packet_queue *q); diff --git a/deps/libff/libff/ff-video-decoder.c b/deps/libff/libff/ff-video-decoder.c index ce4ae0a1f..ce1c0987d 100644 --- a/deps/libff/libff/ff-video-decoder.c +++ b/deps/libff/libff/ff-video-decoder.c @@ -68,7 +68,7 @@ void *ff_video_decoder_thread(void *opaque_video_decoder) { struct ff_decoder *decoder = (struct ff_decoder*)opaque_video_decoder; - AVPacket packet = {0}; + struct ff_packet packet = {0}; int complete; AVFrame *frame = av_frame_alloc(); int ret; @@ -80,13 +80,13 @@ void *ff_video_decoder_thread(void *opaque_video_decoder) break; } - if (packet.data == decoder->packet_queue.flush_packet.data) { + if (packet.base.data == decoder->packet_queue.flush_packet.base.data) { avcodec_flush_buffers(decoder->codec); continue; } avcodec_decode_video2(decoder->codec, frame, - &complete, &packet); + &complete, &packet.base); // Did we get an entire video frame? This doesn't guarantee // there is a picture to show for some codecs, but we still want @@ -104,7 +104,7 @@ void *ff_video_decoder_thread(void *opaque_video_decoder) av_frame_unref(frame); } - av_free_packet(&packet); + av_free_packet(&packet.base); } av_frame_free(&frame); From 2b3d82aeac821cb90ae78653a491a256cc47aa15 Mon Sep 17 00:00:00 2001 From: John Bradley Date: Thu, 12 Mar 2015 14:38:33 -0500 Subject: [PATCH 03/23] deps-libff: Add flag whether a decoder is hardware accelerated This lets the decoder make decisions based on whether it is a hardware decoder or not. Specifically, hardware decoders are more strict as to which frames can be dropped in an h264 stream. --- deps/libff/libff/ff-decoder.h | 1 + deps/libff/libff/ff-demuxer.c | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/deps/libff/libff/ff-decoder.h b/deps/libff/libff/ff-decoder.h index 4f2a75176..089e01d37 100644 --- a/deps/libff/libff/ff-decoder.h +++ b/deps/libff/libff/ff-decoder.h @@ -43,6 +43,7 @@ struct ff_decoder { double current_pts; // pts of the most recently dispatched frame int64_t current_pts_time; // clock time when current_pts was set + bool hwaccel_decoder; struct ff_clock *clock; enum ff_av_sync_type natural_sync_clock; diff --git a/deps/libff/libff/ff-demuxer.c b/deps/libff/libff/ff-demuxer.c index 55eb7d286..dfea1a1c8 100644 --- a/deps/libff/libff/ff-demuxer.c +++ b/deps/libff/libff/ff-demuxer.c @@ -183,7 +183,8 @@ enum AVPixelFormat get_hwaccel_format(struct AVCodecContext *s, } static bool initialize_decoder(struct ff_demuxer *demuxer, - AVCodecContext *codec_context, AVStream *stream) + AVCodecContext *codec_context, AVStream *stream, + bool hwaccel_decoder) { switch (codec_context->codec_type) { case AVMEDIA_TYPE_AUDIO: @@ -192,6 +193,7 @@ static bool initialize_decoder(struct ff_demuxer *demuxer, demuxer->options.audio_packet_queue_size, demuxer->options.audio_frame_queue_size); + demuxer->audio_decoder->hwaccel_decoder = hwaccel_decoder; demuxer->audio_decoder->natural_sync_clock = AV_SYNC_AUDIO_MASTER; demuxer->audio_decoder->clock = &demuxer->clock; @@ -214,6 +216,7 @@ static bool initialize_decoder(struct ff_demuxer *demuxer, demuxer->options.video_packet_queue_size, demuxer->options.video_frame_queue_size); + demuxer->video_decoder->hwaccel_decoder = hwaccel_decoder; demuxer->video_decoder->natural_sync_clock = AV_SYNC_VIDEO_MASTER; demuxer->video_decoder->clock = &demuxer->clock; @@ -240,6 +243,7 @@ static bool find_decoder(struct ff_demuxer *demuxer, AVStream *stream) AVDictionary *options_dict = NULL; int ret; + bool hwaccel_decoder = false; codec_context = stream->codec; // enable reference counted frames since we may have a buffer size @@ -269,6 +273,7 @@ static bool find_decoder(struct ff_demuxer *demuxer, AVStream *stream) codec_context->codec_id); } else { codec = codec_vda; + hwaccel_decoder = true; } } } @@ -290,7 +295,8 @@ static bool find_decoder(struct ff_demuxer *demuxer, AVStream *stream) } } - return initialize_decoder(demuxer, codec_context, stream); + return initialize_decoder(demuxer, codec_context, stream, + hwaccel_decoder); } void ff_demuxer_flush(struct ff_demuxer *demuxer) From de574e99e3b4da10964a1ecce2aa152a17bfca8a Mon Sep 17 00:00:00 2001 From: John Bradley Date: Thu, 12 Mar 2015 14:55:51 -0500 Subject: [PATCH 04/23] deps-libff: Fix bug where rel time was used instead of abs The bug was undetected because it accidentally fell into an error case that slept the correct amount of time. pthread_cond_timedwait takes an absolute time in the future to wait until. The value we were passing was always in the past so it was immediately failing with a TIMEDOUT error code. --- deps/libff/libff/ff-timer.c | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/deps/libff/libff/ff-timer.c b/deps/libff/libff/ff-timer.c index 5fd9d663c..804657102 100644 --- a/deps/libff/libff/ff-timer.c +++ b/deps/libff/libff/ff-timer.c @@ -36,20 +36,18 @@ static void *timer_thread(void *opaque) uint64_t current_time = av_gettime(); if (current_time < timer->next_wake) { - int64_t sleep_time_us = timer->next_wake - current_time; - - struct timespec sleep_time; - sleep_time.tv_sec = - (long)sleep_time_us / AV_TIME_BASE; - sleep_time.tv_nsec = - (long)(sleep_time_us % AV_TIME_BASE) - * 1000; + struct timespec sleep_time = { + .tv_sec = timer->next_wake / AV_TIME_BASE, + .tv_nsec = (timer->next_wake % AV_TIME_BASE) + * 1000 + }; ret = pthread_cond_timedwait(&timer->cond, &timer->mutex, &sleep_time); if (ret != 0) { // failed to wait, just sleep - av_usleep((unsigned)sleep_time_us); + av_usleep((unsigned)(timer->next_wake + - current_time)); } // we can be woken up merely to set a sooner wake time From d9fe44f021d486b8f55e08270ce52261fc27e811 Mon Sep 17 00:00:00 2001 From: John Bradley Date: Thu, 12 Mar 2015 15:09:38 -0500 Subject: [PATCH 05/23] deps-libff: Only sleep if the timed wait didn't expire --- deps/libff/libff/ff-timer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/libff/libff/ff-timer.c b/deps/libff/libff/ff-timer.c index 804657102..e58493b25 100644 --- a/deps/libff/libff/ff-timer.c +++ b/deps/libff/libff/ff-timer.c @@ -44,7 +44,7 @@ static void *timer_thread(void *opaque) ret = pthread_cond_timedwait(&timer->cond, &timer->mutex, &sleep_time); - if (ret != 0) { + if (ret != ETIMEDOUT) { // failed to wait, just sleep av_usleep((unsigned)(timer->next_wake - current_time)); From 7a4a5e3fadf76eda0fe93496b730164cc82807f5 Mon Sep 17 00:00:00 2001 From: John Bradley Date: Thu, 12 Mar 2015 15:15:52 -0500 Subject: [PATCH 06/23] deps-libff: Move timer callback outside of lock --- deps/libff/libff/ff-timer.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/deps/libff/libff/ff-timer.c b/deps/libff/libff/ff-timer.c index e58493b25..a7ddc1bea 100644 --- a/deps/libff/libff/ff-timer.c +++ b/deps/libff/libff/ff-timer.c @@ -27,6 +27,7 @@ static void *timer_thread(void *opaque) int ret; while (true) { + bool callback = false; pthread_mutex_lock(&timer->mutex); if (timer->abort) { @@ -60,11 +61,14 @@ static void *timer_thread(void *opaque) // we woke up for some reason current_time = av_gettime(); if (timer->next_wake <= current_time || timer->needs_wake) { - timer->callback(timer->opaque); + callback = true; timer->needs_wake = false; } pthread_mutex_unlock(&timer->mutex); + + if (callback) + timer->callback(timer->opaque); } return NULL; From 47783f26c726595997876007a5e6de7df9967c11 Mon Sep 17 00:00:00 2001 From: John Bradley Date: Thu, 12 Mar 2015 15:16:55 -0500 Subject: [PATCH 07/23] deps-libff: Abort timer thread instead of cancelling --- deps/libff/libff/ff-timer.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/deps/libff/libff/ff-timer.c b/deps/libff/libff/ff-timer.c index a7ddc1bea..61064ca2e 100644 --- a/deps/libff/libff/ff-timer.c +++ b/deps/libff/libff/ff-timer.c @@ -99,7 +99,11 @@ void ff_timer_free(struct ff_timer *timer) assert(timer != NULL); - pthread_cancel(timer->timer_thread); + pthread_mutex_lock(&timer->mutex); + timer->abort = true; + pthread_cond_signal(&timer->cond); + pthread_mutex_unlock(&timer->mutex); + pthread_join(timer->timer_thread, &thread_result); pthread_mutex_destroy(&timer->mutex); From 164cbeeede9e0bd6b06ef5b89f2af83dbae4633d Mon Sep 17 00:00:00 2001 From: kc5nra Date: Thu, 19 Mar 2015 00:25:58 -0500 Subject: [PATCH 08/23] deps-libff: Add atomic long inc/dec functions --- deps/libff/CMakeLists.txt | 9 +++++++ deps/libff/libff/ff-threading-posix.c | 27 +++++++++++++++++++++ deps/libff/libff/ff-threading-windows.c | 31 +++++++++++++++++++++++++ deps/libff/libff/ff-threading.h | 20 ++++++++++++++++ 4 files changed, 87 insertions(+) create mode 100644 deps/libff/libff/ff-threading-posix.c create mode 100644 deps/libff/libff/ff-threading-windows.c create mode 100644 deps/libff/libff/ff-threading.h diff --git a/deps/libff/CMakeLists.txt b/deps/libff/CMakeLists.txt index d6f6cb1a1..72d52027a 100644 --- a/deps/libff/CMakeLists.txt +++ b/deps/libff/CMakeLists.txt @@ -17,6 +17,7 @@ set(libff_HEADERS libff/ff-clock.h libff/ff-frame.h libff/ff-packet-queue.h + libff/ff-threading.h libff/ff-timer.h # libff/ff-demuxer.h @@ -36,6 +37,14 @@ set(libff_SOURCES libff/ff-audio-decoder.c libff/ff-video-decoder.c) +if (WIN32) + list(APPEND libff_SOURCES + libff/ff-threading-windows.c) +else(WIN32) + list(APPEND libff_SOURCES + libff/ff-threading-posix.c) +endif(WIN32) + add_library (libff STATIC ${libff_HEADERS} ${libff_SOURCES}) diff --git a/deps/libff/libff/ff-threading-posix.c b/deps/libff/libff/ff-threading-posix.c new file mode 100644 index 000000000..1491fd6fd --- /dev/null +++ b/deps/libff/libff/ff-threading-posix.c @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2014 Hugh Bailey + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include "ff-threading.h" + +long ff_atomic_inc_long(volatile long *val) +{ + return __sync_add_and_fetch(val, 1); +} + +long ff_atomic_dec_long(volatile long *val) +{ + return __sync_sub_and_fetch(val, 1); +} diff --git a/deps/libff/libff/ff-threading-windows.c b/deps/libff/libff/ff-threading-windows.c new file mode 100644 index 000000000..56b034e0d --- /dev/null +++ b/deps/libff/libff/ff-threading-windows.c @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2014 Hugh Bailey + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include "ff-threading.h" + +#define WIN32_LEAN_AND_MEAN +#include +#undef WIN32_LEAN_AND_MEAN + +long ff_atomic_inc_long(volatile long *val) +{ + return InterlockedIncrement(val); +} + +long ff_atomic_dec_long(volatile long *val) +{ + return InterlockedDecrement(val); +} diff --git a/deps/libff/libff/ff-threading.h b/deps/libff/libff/ff-threading.h new file mode 100644 index 000000000..c5f3fc693 --- /dev/null +++ b/deps/libff/libff/ff-threading.h @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2015 John R. Bradley + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#pragma once + +long ff_atomic_inc_long(volatile long *val); +long ff_atomic_dec_long(volatile long *val); From 5b3190593c229d282b7342a6d7572ecbbf4ceb64 Mon Sep 17 00:00:00 2001 From: John Bradley Date: Thu, 12 Mar 2015 15:24:15 -0500 Subject: [PATCH 09/23] deps-libff: Add reference counting to clock Add referencing counting to determine when to release a clock Due to no fixed ownership of clocks (packets, frames, decoders and refresh thread all may own a clock at some point). --- deps/libff/libff/ff-clock.c | 52 +++++++++++++++++++++++++++++++++++++ deps/libff/libff/ff-clock.h | 12 +++++++++ 2 files changed, 64 insertions(+) diff --git a/deps/libff/libff/ff-clock.c b/deps/libff/libff/ff-clock.c index 6939d47b7..b22a03ce4 100644 --- a/deps/libff/libff/ff-clock.c +++ b/deps/libff/libff/ff-clock.c @@ -15,8 +15,60 @@ */ #include "ff-clock.h" +#include "ff-threading.h" + +#include double ff_get_sync_clock(struct ff_clock *clock) { return clock->sync_clock(clock->opaque); } + +struct ff_clock *ff_clock_init(struct ff_clock *clock) +{ + clock = av_mallocz(sizeof(struct ff_clock)); + + if (clock == NULL) + return NULL; + + if (pthread_mutex_init(&clock->mutex, NULL) != 0) + goto fail; + + if (pthread_cond_init(&clock->cond, NULL) != 0) + goto fail1; + + return clock; + +fail1: + pthread_mutex_destroy(&clock->mutex); +fail: + av_free(clock); + + return NULL; +} + +struct ff_clock *ff_clock_retain(struct ff_clock *clock) +{ + ff_atomic_inc_long(&clock->retain); + + return clock; +} + +struct ff_clock *ff_clock_move(struct ff_clock **clock) +{ + struct ff_clock *retained_clock = ff_clock_retain(*clock); + ff_clock_release(clock); + + return retained_clock; +} + +void ff_clock_release(struct ff_clock **clock) +{ + if (ff_atomic_dec_long(&(*clock)->retain) == 0) { + pthread_cond_destroy(&(*clock)->cond); + pthread_mutex_destroy(&(*clock)->mutex); + av_free(*clock); + } + + *clock = NULL; +} diff --git a/deps/libff/libff/ff-clock.h b/deps/libff/libff/ff-clock.h index 5f8d4f49d..608eb6971 100644 --- a/deps/libff/libff/ff-clock.h +++ b/deps/libff/libff/ff-clock.h @@ -19,6 +19,9 @@ #define AV_SYNC_THRESHOLD 0.01 #define AV_NOSYNC_THRESHOLD 10.0 +#include +#include + enum ff_av_sync_type { AV_SYNC_AUDIO_MASTER, AV_SYNC_VIDEO_MASTER, @@ -30,9 +33,18 @@ typedef double (*ff_sync_clock)(void *opaque); struct ff_clock { ff_sync_clock sync_clock; enum ff_av_sync_type sync_type; + + pthread_mutex_t mutex; + pthread_cond_t cond; + volatile long retain; + void *opaque; }; typedef struct ff_clock ff_clock_t; +struct ff_clock * ff_clock_init(); double ff_get_sync_clock(struct ff_clock *clock); +struct ff_clock *ff_clock_retain(struct ff_clock *clock); +struct ff_clock *ff_clock_move(struct ff_clock **clock); +void ff_clock_release(struct ff_clock **clock); From 6e42f38386c6e2d258e048329ee388f863b8bb8b Mon Sep 17 00:00:00 2001 From: John Bradley Date: Thu, 12 Mar 2015 15:27:15 -0500 Subject: [PATCH 10/23] deps-libff: Add master/slave deferred clock methods Enables clocks to wait if the main sync clock has not been started yet. An example of this is a video stream (video/audio) being synced to the video clock. If an audio frame gets produced before the video clock has started it will wait. --- deps/libff/libff/ff-clock.c | 66 +++++++++++++++++++++++++++++++++++++ deps/libff/libff/ff-clock.h | 7 ++++ 2 files changed, 73 insertions(+) diff --git a/deps/libff/libff/ff-clock.c b/deps/libff/libff/ff-clock.c index b22a03ce4..79db93aa0 100644 --- a/deps/libff/libff/ff-clock.c +++ b/deps/libff/libff/ff-clock.c @@ -18,12 +18,78 @@ #include "ff-threading.h" #include +#include + +// Amount of microseconds between checks to see if a clock has started +#define CLOCK_START_CHECK_INTERVAL 100 double ff_get_sync_clock(struct ff_clock *clock) { return clock->sync_clock(clock->opaque); } +int64_t ff_clock_start_time(struct ff_clock *clock) +{ + int64_t start_time = AV_NOPTS_VALUE; + + pthread_mutex_lock(&clock->mutex); + if (clock->started) + start_time = clock->start_time; + pthread_mutex_unlock(&clock->mutex); + + return start_time; +} + +bool ff_clock_start(struct ff_clock *clock, enum ff_av_sync_type sync_type, + const bool *abort) +{ + bool release = false; + bool aborted = false; + + if (clock->sync_type == sync_type && !clock->started) { + pthread_mutex_lock(&clock->mutex); + if (!clock->started) { + clock->start_time = av_gettime(); + clock->started = true; + } + pthread_cond_signal(&clock->cond); + pthread_mutex_unlock(&clock->mutex); + } else { + while (!clock->started) { + pthread_mutex_lock(&clock->mutex); + int64_t current_time = av_gettime() + + CLOCK_START_CHECK_INTERVAL; + struct timespec sleep_time = { + .tv_sec = current_time / AV_TIME_BASE, + .tv_nsec = (current_time % AV_TIME_BASE) * 1000 + }; + pthread_cond_timedwait(&clock->cond, &clock->mutex, + &sleep_time); + + aborted = *abort; + + // There is no way anyone can signal us since we are + // the only reference + if (clock->retain == 1) + release = true; + pthread_mutex_unlock(&clock->mutex); + + if (aborted || release) { + av_log(NULL, AV_LOG_ERROR, "could not start " + "slave clock as master clock " + "was never started before " + "being released or aborted"); + break; + } + } + } + + if (release) + ff_clock_release(&clock); + + return !release && !aborted; +} + struct ff_clock *ff_clock_init(struct ff_clock *clock) { clock = av_mallocz(sizeof(struct ff_clock)); diff --git a/deps/libff/libff/ff-clock.h b/deps/libff/libff/ff-clock.h index 608eb6971..07c83d4ba 100644 --- a/deps/libff/libff/ff-clock.h +++ b/deps/libff/libff/ff-clock.h @@ -20,6 +20,7 @@ #define AV_NOSYNC_THRESHOLD 10.0 #include +#include #include enum ff_av_sync_type { @@ -33,10 +34,12 @@ typedef double (*ff_sync_clock)(void *opaque); struct ff_clock { ff_sync_clock sync_clock; enum ff_av_sync_type sync_type; + uint64_t start_time; pthread_mutex_t mutex; pthread_cond_t cond; volatile long retain; + bool started; void *opaque; }; @@ -48,3 +51,7 @@ double ff_get_sync_clock(struct ff_clock *clock); struct ff_clock *ff_clock_retain(struct ff_clock *clock); struct ff_clock *ff_clock_move(struct ff_clock **clock); void ff_clock_release(struct ff_clock **clock); + +int64_t ff_clock_start_time(struct ff_clock *clock); +bool ff_clock_start(struct ff_clock *clock, enum ff_av_sync_type sync_type, + const bool *abort); From a5e0462a88f6891e45d86643b89d374d36ec2552 Mon Sep 17 00:00:00 2001 From: John Bradley Date: Thu, 12 Mar 2015 15:32:11 -0500 Subject: [PATCH 11/23] deps-libff: set master clock sync type based on default stream --- deps/libff/libff/ff-demuxer.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/deps/libff/libff/ff-demuxer.c b/deps/libff/libff/ff-demuxer.c index dfea1a1c8..03ed44ca3 100644 --- a/deps/libff/libff/ff-demuxer.c +++ b/deps/libff/libff/ff-demuxer.c @@ -360,6 +360,19 @@ static bool find_and_initialize_stream_decoders(struct ff_demuxer *demuxer) audio_stream = format_context->streams[i]; } + int default_stream_index = av_find_default_stream_index( + demuxer->format_context); + + if (default_stream_index >= 0) { + AVStream *stream = + format_context->streams[default_stream_index]; + + if (stream->codec->codec_type == AVMEDIA_TYPE_AUDIO) + demuxer->clock.sync_type = AV_SYNC_AUDIO_MASTER; + else if (stream->codec->codec_type == AVMEDIA_TYPE_VIDEO) + demuxer->clock.sync_type = AV_SYNC_VIDEO_MASTER; + } + if (video_stream != NULL) find_decoder(demuxer, video_stream); From 6b36d39345cda1c0d835d58deb9e25b48d3615fd Mon Sep 17 00:00:00 2001 From: John Bradley Date: Thu, 12 Mar 2015 22:53:29 -0500 Subject: [PATCH 12/23] deps-libff: Add clock chaining to packets/frames This attaches clocks to packets and frames and defers the start time until that particular frame is presented. Any packets/frames in the future with the same clock will reference that start time. This fixes issues when there are multiple start times in a large buffer (looped video/images/audio) and different frames need different reference clocks to present correctly. --- deps/libff/libff/ff-audio-decoder.c | 25 ++++++++++++++++ deps/libff/libff/ff-decoder.c | 44 ++++++++++++++++++++++++----- deps/libff/libff/ff-demuxer.c | 28 +++++++++++++++--- deps/libff/libff/ff-frame.h | 4 ++- deps/libff/libff/ff-packet-queue.c | 2 ++ deps/libff/libff/ff-packet-queue.h | 3 ++ deps/libff/libff/ff-video-decoder.c | 13 +++++++++ 7 files changed, 107 insertions(+), 12 deletions(-) diff --git a/deps/libff/libff/ff-audio-decoder.c b/deps/libff/libff/ff-audio-decoder.c index b31a6633d..ebf8caed1 100644 --- a/deps/libff/libff/ff-audio-decoder.c +++ b/deps/libff/libff/ff-audio-decoder.c @@ -39,6 +39,22 @@ static inline void shrink_packet(struct ff_packet *packet, int packet_length) } } +static bool handle_reset_packet(struct ff_decoder *decoder, + struct ff_packet *packet) +{ + if (decoder->clock != NULL) + ff_clock_release(&decoder->clock); + decoder->clock = packet->clock; + av_free_packet(&packet->base); + + // not a real packet, so try to get another packet + if (packet_queue_get(&decoder->packet_queue, packet, 1) + == FF_PACKET_FAIL) + return false; + + return true; +} + static int decode_frame(struct ff_decoder *decoder, struct ff_packet *packet, AVFrame *frame, bool *frame_complete) { @@ -86,6 +102,11 @@ static int decode_frame(struct ff_decoder *decoder, return -1; } } + + // Packet has a new clock (reset packet) + if (packet->clock != NULL) + if (!handle_reset_packet(decoder, packet)) + return -1; } } @@ -113,6 +134,7 @@ static bool queue_frame(struct ff_decoder *decoder, AVFrame *frame, av_frame_free(&queue_frame->frame); queue_frame->frame = av_frame_clone(frame); + queue_frame->clock = ff_clock_retain(decoder->clock); if (call_initialize) ff_callbacks_frame_initialize(queue_frame, decoder->callbacks); @@ -154,6 +176,9 @@ void *ff_audio_decoder_thread(void *opaque_audio_decoder) av_free_packet(&packet.base); } + if (decoder->clock != NULL) + ff_clock_release(&decoder->clock); + av_frame_free(&frame); return NULL; } diff --git a/deps/libff/libff/ff-decoder.c b/deps/libff/libff/ff-decoder.c index a9fd5fe62..b51d6a81d 100644 --- a/deps/libff/libff/ff-decoder.c +++ b/deps/libff/libff/ff-decoder.c @@ -117,10 +117,13 @@ void ff_decoder_free(struct ff_decoder *decoder) ff_callbacks_frame_free(frame, decoder->callbacks); - if (frame != NULL && frame->frame != NULL) - av_frame_unref(frame->frame); - if (frame != NULL) + if (frame != NULL) { + if (frame->frame != NULL) + av_frame_unref(frame->frame); + if (frame->clock != NULL) + ff_clock_release(&frame->clock); av_free(frame); + } } packet_queue_free(&decoder->packet_queue); @@ -143,11 +146,11 @@ double ff_decoder_clock(void *opaque) return decoder->current_pts + delta; } -static double get_sync_adjusted_pts_diff(struct ff_decoder *decoder, +static double get_sync_adjusted_pts_diff(struct ff_clock *clock, double pts, double pts_diff) { double new_pts_diff = pts_diff; - double sync_time = ff_get_sync_clock(decoder->clock); + double sync_time = ff_get_sync_clock(clock); double diff = pts - sync_time; double sync_threshold; @@ -191,6 +194,29 @@ void ff_decoder_refresh(void *opaque) frame = ff_circular_queue_peek_read( &decoder->frame_queue); + // Get frame clock and start it if needed + ff_clock_t *clock = ff_clock_move(&frame->clock); + if (!ff_clock_start(clock, decoder->natural_sync_clock, + &decoder->refresh_timer.abort)) { + ff_clock_release(&clock); + + // Our clock was never started and deleted or + // aborted + + // Drop this frame? The only way this can happen + // is if one stream finishes before another and + // the input is looping or canceled. Until we + // get another clock we will unable to continue + + ff_decoder_schedule_refresh(decoder, 100); + + // Drop this frame as we have no way of timing + // it + ff_circular_queue_advance_read( + &decoder->frame_queue); + return; + } + decoder->current_pts = frame->pts; decoder->current_pts_time = av_gettime(); @@ -208,9 +234,9 @@ void ff_decoder_refresh(void *opaque) decoder->previous_pts = frame->pts; // if not synced against natural clock - if (decoder->clock->sync_type + if (clock->sync_type != decoder->natural_sync_clock) { - pts_diff = get_sync_adjusted_pts_diff(decoder, + pts_diff = get_sync_adjusted_pts_diff(clock, frame->pts, pts_diff); } @@ -224,6 +250,10 @@ void ff_decoder_refresh(void *opaque) delay_until_next_wake = 0.010L; } + if (delay_until_next_wake > pts_diff) + delay_until_next_wake = pts_diff; + + ff_clock_release(&clock); ff_callbacks_frame(decoder->callbacks, frame); ff_decoder_schedule_refresh(decoder, diff --git a/deps/libff/libff/ff-demuxer.c b/deps/libff/libff/ff-demuxer.c index 03ed44ca3..8b725d22f 100644 --- a/deps/libff/libff/ff-demuxer.c +++ b/deps/libff/libff/ff-demuxer.c @@ -196,8 +196,6 @@ static bool initialize_decoder(struct ff_demuxer *demuxer, demuxer->audio_decoder->hwaccel_decoder = hwaccel_decoder; demuxer->audio_decoder->natural_sync_clock = AV_SYNC_AUDIO_MASTER; - demuxer->audio_decoder->clock = &demuxer->clock; - demuxer->audio_decoder->callbacks = &demuxer->audio_callbacks; if (!ff_callbacks_format(&demuxer->audio_callbacks, @@ -219,8 +217,6 @@ static bool initialize_decoder(struct ff_demuxer *demuxer, demuxer->video_decoder->hwaccel_decoder = hwaccel_decoder; demuxer->video_decoder->natural_sync_clock = AV_SYNC_VIDEO_MASTER; - demuxer->video_decoder->clock = &demuxer->clock; - demuxer->video_decoder->callbacks = &demuxer->video_callbacks; if (!ff_callbacks_format(&demuxer->video_callbacks, @@ -316,6 +312,27 @@ void ff_demuxer_flush(struct ff_demuxer *demuxer) } } +void ff_demuxer_reset(struct ff_demuxer *demuxer) +{ + struct ff_packet *packet = av_mallocz(sizeof(struct ff_packet)); + struct ff_clock *clock = ff_clock_init(); + clock->sync_type = demuxer->clock.sync_type; + clock->sync_clock = demuxer->clock.sync_clock; + clock->opaque = demuxer->clock.opaque; + + packet->clock = clock; + + if (demuxer->audio_decoder != NULL) { + packet_queue_put(&demuxer->audio_decoder->packet_queue, packet); + ff_clock_retain(clock); + } + + if (demuxer->video_decoder != NULL) { + packet_queue_put(&demuxer->video_decoder->packet_queue, packet); + ff_clock_retain(clock); + } +} + static bool open_input(struct ff_demuxer *demuxer, AVFormatContext **format_context) { @@ -446,6 +463,7 @@ static bool handle_seek(struct ff_demuxer *demuxer) } else { if (demuxer->seek_flush) ff_demuxer_flush(demuxer); + ff_demuxer_reset(demuxer); } demuxer->seek_request = false; @@ -477,6 +495,8 @@ static void *demux_thread(void *opaque) if (!find_and_initialize_stream_decoders(demuxer)) goto fail; + ff_demuxer_reset(demuxer); + while (!demuxer->abort) { // failed to seek (looping?) if (!handle_seek(demuxer)) diff --git a/deps/libff/libff/ff-frame.h b/deps/libff/libff/ff-frame.h index 091967cac..4c7f91d1f 100644 --- a/deps/libff/libff/ff-frame.h +++ b/deps/libff/libff/ff-frame.h @@ -16,11 +16,13 @@ #pragma once +#include "ff-clock.h" + #include struct ff_frame { AVFrame *frame; - void *opaque; + struct ff_clock *clock; double pts; int64_t duration; }; diff --git a/deps/libff/libff/ff-packet-queue.c b/deps/libff/libff/ff-packet-queue.c index 4bf63d379..bec472fa9 100644 --- a/deps/libff/libff/ff-packet-queue.c +++ b/deps/libff/libff/ff-packet-queue.c @@ -147,6 +147,8 @@ void packet_queue_flush(struct ff_packet_queue *q) packet = q->first_packet) { q->first_packet = packet->next; av_free_packet(&packet->packet.base); + if (packet->packet.clock != NULL) + ff_clock_release(&packet->packet.clock); av_freep(&packet); } diff --git a/deps/libff/libff/ff-packet-queue.h b/deps/libff/libff/ff-packet-queue.h index 26e00dcf6..2bbf8e2aa 100644 --- a/deps/libff/libff/ff-packet-queue.h +++ b/deps/libff/libff/ff-packet-queue.h @@ -16,6 +16,8 @@ #pragma once +#include "ff-clock.h" + #include #include #include @@ -26,6 +28,7 @@ struct ff_packet { AVPacket base; + ff_clock_t *clock; }; struct ff_packet_list { diff --git a/deps/libff/libff/ff-video-decoder.c b/deps/libff/libff/ff-video-decoder.c index ce1c0987d..c22381ad0 100644 --- a/deps/libff/libff/ff-video-decoder.c +++ b/deps/libff/libff/ff-video-decoder.c @@ -53,6 +53,7 @@ static bool queue_frame(struct ff_decoder *decoder, AVFrame *frame, av_frame_free(&queue_frame->frame); queue_frame->frame = av_frame_clone(frame); + queue_frame->clock = ff_clock_retain(decoder->clock); if (call_initialize) ff_callbacks_frame_initialize(queue_frame, decoder->callbacks); @@ -85,6 +86,15 @@ void *ff_video_decoder_thread(void *opaque_video_decoder) continue; } + // We received a reset packet with a new clock + if (packet.clock != NULL) { + if (decoder->clock != NULL) + ff_clock_release(&decoder->clock); + decoder->clock = ff_clock_move(&packet.clock); + av_free_packet(&packet.base); + continue; + } + avcodec_decode_video2(decoder->codec, frame, &complete, &packet.base); @@ -107,6 +117,9 @@ void *ff_video_decoder_thread(void *opaque_video_decoder) av_free_packet(&packet.base); } + if (decoder->clock != NULL) + ff_clock_release(&decoder->clock); + av_frame_free(&frame); return NULL; } From 3628d3402da1ea4f3357b19785ca46f620ac21e4 Mon Sep 17 00:00:00 2001 From: John Bradley Date: Thu, 12 Mar 2015 16:15:01 -0500 Subject: [PATCH 13/23] deps-libff: Add frame dropping This, if set, instructs the decoders to drop frames if a specific timing window is not met. --- deps/libff/libff/ff-audio-decoder.c | 13 +++++++++++++ deps/libff/libff/ff-decoder.c | 27 +++++++++++++++++++++++++++ deps/libff/libff/ff-decoder.h | 4 ++++ deps/libff/libff/ff-demuxer.c | 4 ++++ deps/libff/libff/ff-demuxer.h | 1 + deps/libff/libff/ff-video-decoder.c | 15 +++++++++++++++ 6 files changed, 64 insertions(+) diff --git a/deps/libff/libff/ff-audio-decoder.c b/deps/libff/libff/ff-audio-decoder.c index ebf8caed1..f951fdd18 100644 --- a/deps/libff/libff/ff-audio-decoder.c +++ b/deps/libff/libff/ff-audio-decoder.c @@ -55,6 +55,17 @@ static bool handle_reset_packet(struct ff_decoder *decoder, return true; } +static void drop_late_packets(struct ff_decoder *decoder, + struct ff_packet *packet) +{ + int64_t start_time = ff_clock_start_time(decoder->clock); + if (start_time != AV_NOPTS_VALUE) { + if (ff_decoder_set_frame_drop_state(decoder, start_time, + packet->base.pts)) + shrink_packet(packet, packet->base.size); + } +} + static int decode_frame(struct ff_decoder *decoder, struct ff_packet *packet, AVFrame *frame, bool *frame_complete) { @@ -65,6 +76,8 @@ static int decode_frame(struct ff_decoder *decoder, while (packet->base.size > 0) { int complete; + drop_late_packets(decoder, packet); + packet_length = avcodec_decode_audio4(decoder->codec, frame, &complete, &packet->base); diff --git a/deps/libff/libff/ff-decoder.c b/deps/libff/libff/ff-decoder.c index b51d6a81d..fb3d9fe39 100644 --- a/deps/libff/libff/ff-decoder.c +++ b/deps/libff/libff/ff-decoder.c @@ -320,3 +320,30 @@ double ff_decoder_get_best_effort_pts(struct ff_decoder *decoder, return d_pts; } + +bool ff_decoder_set_frame_drop_state(struct ff_decoder *decoder, + int64_t start_time, int64_t pts) +{ + if (pts != AV_NOPTS_VALUE) { + int64_t rescaled_pts = av_rescale_q(pts, + decoder->stream->time_base, AV_TIME_BASE_Q); + int64_t master_clock = av_gettime() - + start_time; + + int64_t diff = master_clock - rescaled_pts; + + if (diff > (AV_TIME_BASE / 2)) { + decoder->codec->skip_frame = decoder->frame_drop; + decoder->codec->skip_idct = decoder->frame_drop; + decoder->codec->skip_loop_filter = decoder->frame_drop; + return true; + } else { + decoder->codec->skip_frame = AVDISCARD_DEFAULT; + decoder->codec->skip_idct = AVDISCARD_DEFAULT; + decoder->codec->skip_loop_filter = AVDISCARD_DEFAULT; + return false; + } + } + + return false; +} diff --git a/deps/libff/libff/ff-decoder.h b/deps/libff/libff/ff-decoder.h index 089e01d37..6fb2bf012 100644 --- a/deps/libff/libff/ff-decoder.h +++ b/deps/libff/libff/ff-decoder.h @@ -44,6 +44,7 @@ struct ff_decoder { int64_t current_pts_time; // clock time when current_pts was set bool hwaccel_decoder; + enum AVDiscard frame_drop; struct ff_clock *clock; enum ff_av_sync_type natural_sync_clock; @@ -69,3 +70,6 @@ void ff_decoder_refresh(void *opaque); double ff_decoder_get_best_effort_pts(struct ff_decoder *decoder, AVFrame *frame); + +bool ff_decoder_set_frame_drop_state(struct ff_decoder *decoder, + int64_t start_time, int64_t pts); diff --git a/deps/libff/libff/ff-demuxer.c b/deps/libff/libff/ff-demuxer.c index 8b725d22f..44847af81 100644 --- a/deps/libff/libff/ff-demuxer.c +++ b/deps/libff/libff/ff-demuxer.c @@ -194,6 +194,8 @@ static bool initialize_decoder(struct ff_demuxer *demuxer, demuxer->options.audio_frame_queue_size); demuxer->audio_decoder->hwaccel_decoder = hwaccel_decoder; + demuxer->audio_decoder->frame_drop = + demuxer->options.frame_drop; demuxer->audio_decoder->natural_sync_clock = AV_SYNC_AUDIO_MASTER; demuxer->audio_decoder->callbacks = &demuxer->audio_callbacks; @@ -215,6 +217,8 @@ static bool initialize_decoder(struct ff_demuxer *demuxer, demuxer->options.video_frame_queue_size); demuxer->video_decoder->hwaccel_decoder = hwaccel_decoder; + demuxer->video_decoder->frame_drop = + demuxer->options.frame_drop; demuxer->video_decoder->natural_sync_clock = AV_SYNC_VIDEO_MASTER; demuxer->video_decoder->callbacks = &demuxer->video_callbacks; diff --git a/deps/libff/libff/ff-demuxer.h b/deps/libff/libff/ff-demuxer.h index 938114f0a..d6b20703f 100644 --- a/deps/libff/libff/ff-demuxer.h +++ b/deps/libff/libff/ff-demuxer.h @@ -35,6 +35,7 @@ struct ff_demuxer_options int video_frame_queue_size; bool is_hw_decoding; bool is_looping; + enum AVDiscard frame_drop; }; typedef struct ff_demuxer_options ff_demuxer_options_t; diff --git a/deps/libff/libff/ff-video-decoder.c b/deps/libff/libff/ff-video-decoder.c index c22381ad0..0f1b35f76 100644 --- a/deps/libff/libff/ff-video-decoder.c +++ b/deps/libff/libff/ff-video-decoder.c @@ -73,6 +73,7 @@ void *ff_video_decoder_thread(void *opaque_video_decoder) int complete; AVFrame *frame = av_frame_alloc(); int ret; + bool key_frame; while (!decoder->abort) { ret = packet_queue_get(&decoder->packet_queue, &packet, 1); @@ -95,6 +96,20 @@ void *ff_video_decoder_thread(void *opaque_video_decoder) continue; } + int64_t start_time = ff_clock_start_time(decoder->clock); + key_frame = packet.base.flags & AV_PKT_FLAG_KEY; + + // We can only make decisions on keyframes for + // hw decoders (maybe just OSX?) + // For now, always make drop decisions on keyframes + bool frame_drop_check = key_frame; + // Must have a proper packet pts to drop frames here + frame_drop_check &= start_time != AV_NOPTS_VALUE; + + if (frame_drop_check) + ff_decoder_set_frame_drop_state(decoder, + start_time, packet.base.pts); + avcodec_decode_video2(decoder->codec, frame, &complete, &packet.base); From 17f8e3c58bc48988bd25ef39c8d744690809832b Mon Sep 17 00:00:00 2001 From: John Bradley Date: Thu, 12 Mar 2015 16:17:13 -0500 Subject: [PATCH 14/23] obs-ffmpeg: Add frame drop controls to the ffmpeg source UI --- plugins/obs-ffmpeg/data/locale/en-US.ini | 8 ++++++ plugins/obs-ffmpeg/obs-ffmpeg-source.c | 33 ++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/plugins/obs-ffmpeg/data/locale/en-US.ini b/plugins/obs-ffmpeg/data/locale/en-US.ini index df3d306e1..e8c7a0d69 100644 --- a/plugins/obs-ffmpeg/data/locale/en-US.ini +++ b/plugins/obs-ffmpeg/data/locale/en-US.ini @@ -12,3 +12,11 @@ HardwareDecode="Use hardware decoding when available" Advanced="Advanced" AudioBufferSize="Audio Buffer Size (frames)" VideoBufferSize="Video Buffer Size (frames)" +FrameDropping="Frame Dropping Level" +DiscardNone="None" +DiscardDefault="Default (Invalid Packets)" +DiscardNonRef="Non-Reference Frames" +DiscardBiDir="Bi-Directional Frames" +DiscardNonIntra="Non-Intra Frames" +DiscardNonKey="Non-Key Frames" +DiscardAll="All Frames (Careful!)" diff --git a/plugins/obs-ffmpeg/obs-ffmpeg-source.c b/plugins/obs-ffmpeg/obs-ffmpeg-source.c index 11faa8b72..da22f3f28 100644 --- a/plugins/obs-ffmpeg/obs-ffmpeg-source.c +++ b/plugins/obs-ffmpeg/obs-ffmpeg-source.c @@ -266,8 +266,10 @@ static bool is_advanced_modified(obs_properties_t *props, bool enabled = obs_data_get_bool(settings, "advanced"); obs_property_t *abuf = obs_properties_get(props, "audio_buffer_size"); obs_property_t *vbuf = obs_properties_get(props, "video_buffer_size"); + obs_property_t *frame_drop = obs_properties_get(props, "frame_drop"); obs_property_set_visible(abuf, enabled); obs_property_set_visible(vbuf, enabled); + obs_property_set_visible(frame_drop, enabled); return true; } @@ -317,6 +319,27 @@ static obs_properties_t *ffmpeg_source_getproperties(void *data) obs_property_set_visible(prop, false); + prop = obs_properties_add_list(props, "frame_drop", + obs_module_text("FrameDropping"), OBS_COMBO_TYPE_LIST, + OBS_COMBO_FORMAT_INT); + + obs_property_list_add_int(prop, obs_module_text("DiscardNone"), + AVDISCARD_NONE); + obs_property_list_add_int(prop, obs_module_text("DiscardDefault"), + AVDISCARD_DEFAULT); + obs_property_list_add_int(prop, obs_module_text("DiscardNonRef"), + AVDISCARD_NONREF); + obs_property_list_add_int(prop, obs_module_text("DiscardBiDir"), + AVDISCARD_BIDIR); + obs_property_list_add_int(prop, obs_module_text("DiscardNonIntra"), + AVDISCARD_NONINTRA); + obs_property_list_add_int(prop, obs_module_text("DiscardNonKey"), + AVDISCARD_NONKEY); + obs_property_list_add_int(prop, obs_module_text("DiscardAll"), + AVDISCARD_ALL); + + obs_property_set_visible(prop, false); + return props; } @@ -357,6 +380,10 @@ static void ffmpeg_source_update(void *data, obs_data_t *settings) "audio_buffer_size"); int video_buffer_size = (int)obs_data_get_int(settings, "video_buffer_size"); + enum AVDiscard frame_drop = + (enum AVDiscard)obs_data_get_int(settings, + "frame_drop"); + if (audio_buffer_size < 1) { audio_buffer_size = 1; blog(LOG_WARNING, "invalid audio_buffer_size %d", @@ -369,6 +396,12 @@ static void ffmpeg_source_update(void *data, obs_data_t *settings) } s->demuxer->options.audio_frame_queue_size = audio_buffer_size; s->demuxer->options.video_frame_queue_size = video_buffer_size; + + if (frame_drop < AVDISCARD_NONE || frame_drop > AVDISCARD_ALL) { + frame_drop = AVDISCARD_NONE; + blog(LOG_WARNING, "invalid frame_drop %d", frame_drop); + } + s->demuxer->options.frame_drop = frame_drop; } ff_demuxer_set_callbacks(&s->demuxer->video_callbacks, From 312d59da02459d3f2aa7a3266ca7a66ebbfce744 Mon Sep 17 00:00:00 2001 From: John Bradley Date: Thu, 12 Mar 2015 15:59:11 -0500 Subject: [PATCH 15/23] deps-libff: Fix comment formatting --- deps/libff/libff/ff-decoder.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/deps/libff/libff/ff-decoder.c b/deps/libff/libff/ff-decoder.c index fb3d9fe39..65199c487 100644 --- a/deps/libff/libff/ff-decoder.c +++ b/deps/libff/libff/ff-decoder.c @@ -178,7 +178,9 @@ void ff_decoder_refresh(void *opaque) if (decoder && decoder->stream) { if (decoder->frame_queue.size == 0) { if (!decoder->eof) { - // We expected a frame, but there were none available + // We expected a frame, but there were none + // available + // Schedule another call as soon as possible ff_decoder_schedule_refresh(decoder, 1); } else { From f8c38d1fcf1fea4b71f44f905497b3a586c5e190 Mon Sep 17 00:00:00 2001 From: John Bradley Date: Thu, 12 Mar 2015 15:59:47 -0500 Subject: [PATCH 16/23] deps-libff: Remove misleading comment --- deps/libff/libff/ff-decoder.c | 1 - 1 file changed, 1 deletion(-) diff --git a/deps/libff/libff/ff-decoder.c b/deps/libff/libff/ff-decoder.c index 65199c487..d7e9577fd 100644 --- a/deps/libff/libff/ff-decoder.c +++ b/deps/libff/libff/ff-decoder.c @@ -248,7 +248,6 @@ void ff_decoder_refresh(void *opaque) delay_until_next_wake = decoder->timer_next_wake - (av_gettime() / 1000000.0L); if (delay_until_next_wake < 0.010L) { - // accellerate next wake up delay_until_next_wake = 0.010L; } From b2d5b47833d341e2ebc1f678a1729e39226c6f5a Mon Sep 17 00:00:00 2001 From: John Bradley Date: Thu, 12 Mar 2015 16:11:13 -0500 Subject: [PATCH 17/23] deps-libff: Remove extra whitespace --- deps/libff/libff/ff-video-decoder.c | 1 - 1 file changed, 1 deletion(-) diff --git a/deps/libff/libff/ff-video-decoder.c b/deps/libff/libff/ff-video-decoder.c index 0f1b35f76..20e00cb1f 100644 --- a/deps/libff/libff/ff-video-decoder.c +++ b/deps/libff/libff/ff-video-decoder.c @@ -117,7 +117,6 @@ void *ff_video_decoder_thread(void *opaque_video_decoder) // there is a picture to show for some codecs, but we still want // to adjust our various internal clocks for the next frame if (complete) { - // If we don't have a good PTS, try to guess based // on last received PTS provided plus prediction // This function returns a pts scaled to stream From 5465fcb4a2791638f84e6de53232260de1293b50 Mon Sep 17 00:00:00 2001 From: John Bradley Date: Thu, 12 Mar 2015 16:21:30 -0500 Subject: [PATCH 18/23] deps-libff: Initialize FFmpeg network If this is omitted and you use an input that requires the network you get a warning message about future versions not automatically doing this for you. --- deps/libff/libff/ff-demuxer.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deps/libff/libff/ff-demuxer.c b/deps/libff/libff/ff-demuxer.c index 44847af81..1d385011a 100644 --- a/deps/libff/libff/ff-demuxer.c +++ b/deps/libff/libff/ff-demuxer.c @@ -40,7 +40,8 @@ struct ff_demuxer *ff_demuxer_init() av_register_all(); avdevice_register_all(); avfilter_register_all(); - + avformat_network_init(); + demuxer = av_mallocz(sizeof(struct ff_demuxer)); if (demuxer == NULL) return NULL; From 52aae96f7417adcb3583a87bcecb0cd17063aa37 Mon Sep 17 00:00:00 2001 From: John Bradley Date: Thu, 12 Mar 2015 14:46:45 -0500 Subject: [PATCH 19/23] obs-ffmpeg: Add alias of CS601 support --- plugins/obs-ffmpeg/obs-ffmpeg-source.c | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/obs-ffmpeg/obs-ffmpeg-source.c b/plugins/obs-ffmpeg/obs-ffmpeg-source.c index da22f3f28..f9ddfca1b 100644 --- a/plugins/obs-ffmpeg/obs-ffmpeg-source.c +++ b/plugins/obs-ffmpeg/obs-ffmpeg-source.c @@ -78,6 +78,7 @@ static bool set_obs_frame_colorprops(struct ff_frame *frame, switch(frame_cs) { case AVCOL_SPC_BT709: obs_cs = VIDEO_CS_709; break; + case AVCOL_SPC_SMPTE170M: case AVCOL_SPC_BT470BG: obs_cs = VIDEO_CS_601; break; case AVCOL_SPC_UNSPECIFIED: obs_cs = VIDEO_CS_DEFAULT; break; default: From c539c16ecc292dadb66edd92268495eb44912163 Mon Sep 17 00:00:00 2001 From: John Bradley Date: Thu, 12 Mar 2015 14:50:58 -0500 Subject: [PATCH 20/23] obs-ffmpeg: Instead of failing with unsupported CS, use default --- plugins/obs-ffmpeg/obs-ffmpeg-source.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/obs-ffmpeg/obs-ffmpeg-source.c b/plugins/obs-ffmpeg/obs-ffmpeg-source.c index f9ddfca1b..0b1d1e700 100644 --- a/plugins/obs-ffmpeg/obs-ffmpeg-source.c +++ b/plugins/obs-ffmpeg/obs-ffmpeg-source.c @@ -84,7 +84,7 @@ static bool set_obs_frame_colorprops(struct ff_frame *frame, default: blog(LOG_WARNING, "frame using an unsupported colorspace %d", frame_cs); - return false; + obs_cs = VIDEO_CS_DEFAULT; } enum video_range_type range; From ef4ee1fd1a20e0cd3b91585e8e188b43f4cc7546 Mon Sep 17 00:00:00 2001 From: jp9000 Date: Thu, 19 Mar 2015 13:42:59 -0700 Subject: [PATCH 21/23] libobs: Implement flags for properties OBS_PROPERTIES_DEFER_UPDATE: Makes it so the properties are not updated until editing of the properties is complete. --- libobs/obs-properties.c | 12 ++++++++++++ libobs/obs-properties.h | 6 ++++++ 2 files changed, 18 insertions(+) diff --git a/libobs/obs-properties.c b/libobs/obs-properties.c index ddffbe00c..cdaf14b26 100644 --- a/libobs/obs-properties.c +++ b/libobs/obs-properties.c @@ -105,6 +105,7 @@ struct obs_property { struct obs_properties { void *param; void (*destroy)(void *param); + uint32_t flags; struct obs_property *first_property; struct obs_property **last; @@ -131,6 +132,17 @@ void obs_properties_set_param(obs_properties_t *props, props->destroy = destroy; } +void obs_properties_set_flags(obs_properties_t *props, uint32_t flags) +{ + if (props) + props->flags = flags; +} + +uint32_t obs_properties_get_flags(obs_properties_t *props) +{ + return props ? props->flags : 0; +} + void *obs_properties_get_param(obs_properties_t *props) { return props ? props->param : NULL; diff --git a/libobs/obs-properties.h b/libobs/obs-properties.h index c522a6073..dcd7c4225 100644 --- a/libobs/obs-properties.h +++ b/libobs/obs-properties.h @@ -38,6 +38,9 @@ extern "C" { #endif +/** Only update when the user presses OK or Apply */ +#define OBS_PROPERTIES_DEFER_UPDATE (1<<0) + enum obs_property_type { OBS_PROPERTY_INVALID, OBS_PROPERTY_BOOL, @@ -92,6 +95,9 @@ EXPORT obs_properties_t *obs_properties_create_param(void *param, void (*destroy)(void *param)); EXPORT void obs_properties_destroy(obs_properties_t *props); +EXPORT void obs_properties_set_flags(obs_properties_t *props, uint32_t flags); +EXPORT uint32_t obs_properties_get_flags(obs_properties_t *props); + EXPORT void obs_properties_set_param(obs_properties_t *props, void *param, void (*destroy)(void *param)); EXPORT void *obs_properties_get_param(obs_properties_t *props); From dc36f6f4208ee1436b0161bee18d3b449bb708af Mon Sep 17 00:00:00 2001 From: jp9000 Date: Thu, 19 Mar 2015 13:44:48 -0700 Subject: [PATCH 22/23] UI: Implement deferred update flag (properties) --- obs/properties-view.cpp | 5 ++++- obs/properties-view.hpp | 4 ++++ obs/window-basic-properties.cpp | 5 +++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/obs/properties-view.cpp b/obs/properties-view.cpp index db8e74a19..5c9fa8ba4 100644 --- a/obs/properties-view.cpp +++ b/obs/properties-view.cpp @@ -50,6 +50,9 @@ void OBSPropertiesView::ReloadProperties() obs_properties_apply_settings(properties.get(), settings); } + uint32_t flags = obs_properties_get_flags(properties.get()); + deferUpdate = (flags & OBS_PROPERTIES_DEFER_UPDATE) != 0; + RefreshProperties(); } @@ -755,7 +758,7 @@ void WidgetInfo::ControlChanged() return; } - if (view->callback) + if (view->callback && !view->deferUpdate) view->callback(view->obj, view->settings); view->SignalChanged(); diff --git a/obs/properties-view.hpp b/obs/properties-view.hpp index 718fa1467..66a51a03a 100644 --- a/obs/properties-view.hpp +++ b/obs/properties-view.hpp @@ -66,6 +66,7 @@ private: std::vector> children; std::string lastFocused; QWidget *lastWidget = nullptr; + bool deferUpdate; QWidget *NewWidget(obs_property_t *prop, QWidget *widget, const char *signal); @@ -106,4 +107,7 @@ public: int minSize = 0); inline obs_data_t *GetSettings() const {return settings;} + + inline void UpdateSettings() {callback(obj, settings);} + inline bool DeferUpdate() const {return deferUpdate;} }; diff --git a/obs/window-basic-properties.cpp b/obs/window-basic-properties.cpp index 3cca06c9c..a70d29a82 100644 --- a/obs/window-basic-properties.cpp +++ b/obs/window-basic-properties.cpp @@ -117,6 +117,9 @@ void OBSBasicProperties::on_buttonBox_clicked(QAbstractButton *button) if (val == QDialogButtonBox::AcceptRole) { acceptClicked = true; close(); + + if (view->DeferUpdate()) + view->UpdateSettings(); } if (val == QDialogButtonBox::RejectRole) { @@ -260,6 +263,8 @@ bool OBSBasicProperties::ConfirmQuit() switch (button) { case QMessageBox::Save: + if (view->DeferUpdate()) + view->UpdateSettings(); // Do nothing because the settings are already updated break; case QMessageBox::Discard: From d69f03c6a54bc724816b0bd190f03100f38612de Mon Sep 17 00:00:00 2001 From: John Bradley Date: Thu, 19 Mar 2015 16:01:10 -0500 Subject: [PATCH 23/23] obs-ffmpeg: Use deferred update flag for properties --- plugins/obs-ffmpeg/obs-ffmpeg-source.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/obs-ffmpeg/obs-ffmpeg-source.c b/plugins/obs-ffmpeg/obs-ffmpeg-source.c index 0b1d1e700..6e2aee54c 100644 --- a/plugins/obs-ffmpeg/obs-ffmpeg-source.c +++ b/plugins/obs-ffmpeg/obs-ffmpeg-source.c @@ -280,6 +280,9 @@ static obs_properties_t *ffmpeg_source_getproperties(void *data) UNUSED_PARAMETER(data); obs_properties_t *props = obs_properties_create(); + + obs_properties_set_flags(props, OBS_PROPERTIES_DEFER_UPDATE); + obs_property_t *prop; // use this when obs allows non-readonly paths prop = obs_properties_add_bool(props, "is_local_file",