From 923f06bfa6ea80a432f182b50bb6dc5256ba2560 Mon Sep 17 00:00:00 2001 From: Colin Edwards Date: Mon, 26 Aug 2019 17:58:20 -0500 Subject: [PATCH] decklink: Add ability to ingest/embed cea 708 captions (This commit also modifies libobs, UI) --- UI/frontend-plugins/CMakeLists.txt | 1 + .../decklink-captions/CMakeLists.txt | 43 +++++ .../decklink-captions/data/.keepme | 0 .../decklink-captions/decklink-captions.cpp | 157 ++++++++++++++++++ .../decklink-captions/decklink-captions.h | 30 ++++ .../decklink-captions/forms/captions.ui | 115 +++++++++++++ libobs/CMakeLists.txt | 6 +- libobs/obs-internal.h | 12 ++ libobs/obs-output.c | 88 +++++++++- libobs/obs-source.c | 50 ++++++ libobs/obs-source.h | 5 + libobs/obs.h | 23 +++ libobs/util/bitstream.c | 52 ++++++ libobs/util/bitstream.h | 29 ++++ plugins/decklink/OBSVideoFrame.cpp | 72 ++++++++ plugins/decklink/OBSVideoFrame.h | 70 ++++++++ plugins/decklink/decklink-device-instance.cpp | 147 +++++++++++++++- plugins/decklink/decklink-device-instance.hpp | 5 + plugins/decklink/decklink-source.cpp | 6 +- plugins/decklink/linux/CMakeLists.txt | 6 +- plugins/decklink/mac/CMakeLists.txt | 8 +- plugins/decklink/platform.hpp | 1 + plugins/decklink/win/CMakeLists.txt | 11 +- plugins/decklink/win/platform.cpp | 10 ++ test/cmocka/CMakeLists.txt | 7 + test/cmocka/test_bitstream.c | 34 ++++ 26 files changed, 964 insertions(+), 24 deletions(-) create mode 100644 UI/frontend-plugins/decklink-captions/CMakeLists.txt create mode 100644 UI/frontend-plugins/decklink-captions/data/.keepme create mode 100644 UI/frontend-plugins/decklink-captions/decklink-captions.cpp create mode 100644 UI/frontend-plugins/decklink-captions/decklink-captions.h create mode 100644 UI/frontend-plugins/decklink-captions/forms/captions.ui create mode 100644 libobs/util/bitstream.c create mode 100644 libobs/util/bitstream.h create mode 100644 plugins/decklink/OBSVideoFrame.cpp create mode 100644 plugins/decklink/OBSVideoFrame.h create mode 100644 test/cmocka/test_bitstream.c diff --git a/UI/frontend-plugins/CMakeLists.txt b/UI/frontend-plugins/CMakeLists.txt index 908b5c768..eb95ba9d5 100644 --- a/UI/frontend-plugins/CMakeLists.txt +++ b/UI/frontend-plugins/CMakeLists.txt @@ -1,2 +1,3 @@ add_subdirectory(decklink-output-ui) add_subdirectory(frontend-tools) +add_subdirectory(decklink-captions) diff --git a/UI/frontend-plugins/decklink-captions/CMakeLists.txt b/UI/frontend-plugins/decklink-captions/CMakeLists.txt new file mode 100644 index 000000000..88b2b390f --- /dev/null +++ b/UI/frontend-plugins/decklink-captions/CMakeLists.txt @@ -0,0 +1,43 @@ +project(decklink-captions) + +if(APPLE) + find_library(COCOA Cocoa) + include_directories(${COCOA}) +endif() + +if(UNIX AND NOT APPLE) + find_package(X11 REQUIRED) + link_libraries(${X11_LIBRARIES}) + include_directories(${X11_INCLUDE_DIR}) +endif() + +set(decklink-captions_HEADERS + decklink-captions.h + ) +set(decklink-captions_SOURCES + decklink-captions.cpp + ) +set(decklink-captions_UI + forms/captions.ui + ) + +if(APPLE) + set(decklink-captions_PLATFORM_LIBS + ${COCOA}) +endif() + +qt5_wrap_ui(decklink-captions_UI_HEADERS + ${decklink-captions_UI}) + +add_library(decklink-captions MODULE + ${decklink-captions_HEADERS} + ${decklink-captions_SOURCES} + ${decklink-captions_UI_HEADERS} + ) +target_link_libraries(decklink-captions + ${frontend-tools_PLATFORM_LIBS} + obs-frontend-api + Qt5::Widgets + libobs) + +install_obs_plugin_with_data(decklink-captions data) diff --git a/UI/frontend-plugins/decklink-captions/data/.keepme b/UI/frontend-plugins/decklink-captions/data/.keepme new file mode 100644 index 000000000..e69de29bb diff --git a/UI/frontend-plugins/decklink-captions/decklink-captions.cpp b/UI/frontend-plugins/decklink-captions/decklink-captions.cpp new file mode 100644 index 000000000..464f59704 --- /dev/null +++ b/UI/frontend-plugins/decklink-captions/decklink-captions.cpp @@ -0,0 +1,157 @@ +#include +#include +#include +#include +#include "decklink-captions.h" + +OBS_DECLARE_MODULE() +OBS_MODULE_USE_DEFAULT_LOCALE("decklink-captons", "en-US") + +struct obs_captions { + std::string source_name; + OBSWeakSource source; + + void start(); + void stop(); + + obs_captions(); + inline ~obs_captions() { stop(); } +}; + +obs_captions::obs_captions() {} + +static obs_captions *captions = nullptr; + +DecklinkCaptionsUI::DecklinkCaptionsUI(QWidget *parent) + : QDialog(parent), ui(new Ui_CaptionsDialog) +{ + ui->setupUi(this); + + setSizeGripEnabled(true); + + setWindowFlags(windowFlags() & ~Qt::WindowContextHelpButtonHint); + + auto cb = [this](obs_source_t *source) { + uint32_t caps = obs_source_get_output_flags(source); + QString name = obs_source_get_name(source); + + if (caps & OBS_SOURCE_CEA_708) + ui->source->addItem(name); + + OBSWeakSource weak = OBSGetWeakRef(source); + if (weak == captions->source) + ui->source->setCurrentText(name); + return true; + }; + + using cb_t = decltype(cb); + + ui->source->blockSignals(true); + ui->source->addItem(QStringLiteral("")); + ui->source->setCurrentIndex(0); + obs_enum_sources( + [](void *data, obs_source_t *source) { + return (*static_cast(data))(source); + }, + &cb); + ui->source->blockSignals(false); +} + +void DecklinkCaptionsUI::on_source_currentIndexChanged(int) +{ + captions->stop(); + + captions->source_name = ui->source->currentText().toUtf8().constData(); + captions->source = GetWeakSourceByName(captions->source_name.c_str()); + + captions->start(); +} + +static void caption_callback(void *param, obs_source_t *source, + const struct obs_source_cea_708 *captions) +{ + obs_output *output = obs_frontend_get_streaming_output(); + if (output) { + if (obs_frontend_streaming_active() && + obs_output_active(output)) { + obs_output_caption(output, captions); + } + obs_output_release(output); + } +} + +void obs_captions::start() +{ + OBSSource s = OBSGetStrongRef(source); + if (!s) { + //warn("Source invalid"); + return; + } + obs_source_add_caption_callback(s, caption_callback, nullptr); +} + +void obs_captions::stop() +{ + OBSSource s = OBSGetStrongRef(source); + if (s) + obs_source_remove_caption_callback(s, caption_callback, + nullptr); +} + +static void save_decklink_caption_data(obs_data_t *save_data, bool saving, + void *) +{ + if (saving) { + obs_data_t *obj = obs_data_create(); + + obs_data_set_string(obj, "source", + captions->source_name.c_str()); + + obs_data_set_obj(save_data, "decklink_captions", obj); + obs_data_release(obj); + } else { + captions->stop(); + + obs_data_t *obj = + obs_data_get_obj(save_data, "decklink_captions"); + if (!obj) + obj = obs_data_create(); + + captions->source_name = obs_data_get_string(obj, "source"); + captions->source = + GetWeakSourceByName(captions->source_name.c_str()); + obs_data_release(obj); + + captions->start(); + } +} + +void addOutputUI(void) +{ + QAction *action = (QAction *)obs_frontend_add_tools_menu_qaction( + obs_module_text("Decklink Captions")); + + captions = new obs_captions; + + auto cb = []() { + obs_frontend_push_ui_translation(obs_module_get_string); + + QWidget *window = (QWidget *)obs_frontend_get_main_window(); + + DecklinkCaptionsUI dialog(window); + dialog.exec(); + + obs_frontend_pop_ui_translation(); + }; + + obs_frontend_add_save_callback(save_decklink_caption_data, nullptr); + + action->connect(action, &QAction::triggered, cb); +} + +bool obs_module_load(void) +{ + addOutputUI(); + + return true; +} diff --git a/UI/frontend-plugins/decklink-captions/decklink-captions.h b/UI/frontend-plugins/decklink-captions/decklink-captions.h new file mode 100644 index 000000000..7554a1346 --- /dev/null +++ b/UI/frontend-plugins/decklink-captions/decklink-captions.h @@ -0,0 +1,30 @@ +#include +#include +#include +#include +#include +#include "ui_captions.h" + +class DecklinkCaptionsUI : public QDialog { + Q_OBJECT +private: +public: + std::unique_ptr ui; + DecklinkCaptionsUI(QWidget *parent); + +public slots: + void on_source_currentIndexChanged(int idx); +}; + +static inline OBSWeakSource GetWeakSourceByName(const char *name) +{ + OBSWeakSource weak; + obs_source_t *source = obs_get_source_by_name(name); + if (source) { + weak = obs_source_get_weak_source(source); + obs_weak_source_release(weak); + obs_source_release(source); + } + + return weak; +} diff --git a/UI/frontend-plugins/decklink-captions/forms/captions.ui b/UI/frontend-plugins/decklink-captions/forms/captions.ui new file mode 100644 index 000000000..f251008eb --- /dev/null +++ b/UI/frontend-plugins/decklink-captions/forms/captions.ui @@ -0,0 +1,115 @@ + + + CaptionsDialog + + + + 0 + 0 + 519 + 104 + + + + Captions + + + + + + Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter + + + + + Captions.Source + + + source + + + + + + + QComboBox::InsertAlphabetically + + + QComboBox::AdjustToContents + + + + + + + + + Qt::Vertical + + + + 0 + 0 + + + + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + OK + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + + + + + accept + clicked() + CaptionsDialog + accept() + + + 268 + 331 + + + 229 + -11 + + + + + diff --git a/libobs/CMakeLists.txt b/libobs/CMakeLists.txt index 9a58322f9..afb625d63 100644 --- a/libobs/CMakeLists.txt +++ b/libobs/CMakeLists.txt @@ -365,7 +365,8 @@ set(libobs_util_SOURCES util/crc32.c util/text-lookup.c util/cf-parser.c - util/profiler.c) + util/profiler.c + util/bitstream.c) set(libobs_util_HEADERS util/curl/curl-helper.h util/sse-intrin.h @@ -392,7 +393,8 @@ set(libobs_util_HEADERS util/lexer.h util/platform.h util/profiler.h - util/profiler.hpp) + util/profiler.hpp + util/bitstream.h) set(libobs_libobs_SOURCES ${libobs_PLATFORM_SOURCES} diff --git a/libobs/obs-internal.h b/libobs/obs-internal.h index 929a096e2..81a989703 100644 --- a/libobs/obs-internal.h +++ b/libobs/obs-internal.h @@ -36,6 +36,8 @@ #include "obs.h" +#include + #define NUM_TEXTURES 2 #define NUM_CHANNELS 3 #define MICROSECOND_DEN 1000000 @@ -587,6 +589,11 @@ struct audio_cb_info { void *param; }; +struct caption_cb_info { + obs_source_caption_t callback; + void *param; +}; + struct obs_source { struct obs_context_data context; struct obs_source_info info; @@ -690,6 +697,9 @@ struct obs_source { uint32_t async_convert_width[MAX_AV_PLANES]; uint32_t async_convert_height[MAX_AV_PLANES]; + pthread_mutex_t caption_cb_mutex; + DARRAY(struct caption_cb_info) caption_cb_list; + /* async video deinterlacing */ uint64_t deinterlace_offset; uint64_t deinterlace_frame_ts; @@ -977,6 +987,8 @@ struct obs_output { struct caption_text *caption_head; struct caption_text *caption_tail; + struct circlebuf caption_data; + bool valid; uint64_t active_delay_ns; diff --git a/libobs/obs-output.c b/libobs/obs-output.c index c37fd12e8..5383065cc 100644 --- a/libobs/obs-output.c +++ b/libobs/obs-output.c @@ -227,6 +227,7 @@ void obs_output_destroy(obs_output_t *output) os_event_destroy(output->reconnect_stop_event); obs_context_data_free(&output->context); circlebuf_free(&output->delay_data); + circlebuf_free(&output->caption_data); if (output->owns_info_id) bfree((void *)output->info.id); if (output->last_error_message) @@ -267,6 +268,10 @@ bool obs_output_actual_start(obs_output_t *output) os_atomic_dec_long(&output->delay_restart_refs); output->caption_timestamp = 0; + + circlebuf_free(&output->caption_data); + circlebuf_init(&output->caption_data); + return success; } @@ -1207,7 +1212,6 @@ static const uint8_t nal_start[4] = {0, 0, 0, 1}; static bool add_caption(struct obs_output *output, struct encoder_packet *out) { struct encoder_packet backup = *out; - caption_frame_t cf; sei_t sei; uint8_t *data; size_t size; @@ -1224,10 +1228,62 @@ static bool add_caption(struct obs_output *output, struct encoder_packet *out) da_push_back_array(out_data, &ref, sizeof(ref)); da_push_back_array(out_data, out->data, out->size); - caption_frame_init(&cf); - caption_frame_from_text(&cf, &output->caption_head->text[0]); + if (output->caption_data.size > 0) { - sei_from_caption_frame(&sei, &cf); + cea708_t cea708; + cea708_init(&cea708, 0); // set up a new popon frame + void *caption_buf = bzalloc(3 * sizeof(uint8_t)); + + while (output->caption_data.size > 0) { + circlebuf_pop_front(&output->caption_data, caption_buf, + 3 * sizeof(uint8_t)); + + if ((((uint8_t *)caption_buf)[0] & 0x3) != 0) { + // only send cea 608 + continue; + } + + uint16_t captionData = ((uint8_t *)caption_buf)[1]; + captionData = captionData << 8; + captionData += ((uint8_t *)caption_buf)[2]; + + // padding + if (captionData == 0x8080) { + continue; + } + + if (captionData == 0) { + continue; + } + + if (!eia608_parity_varify(captionData)) { + continue; + } + + cea708_add_cc_data(&cea708, 1, + ((uint8_t *)caption_buf)[0] & 0x3, + captionData); + } + + bfree(caption_buf); + + sei_message_t *msg = + sei_message_new(sei_type_user_data_registered_itu_t_t35, + 0, CEA608_MAX_SIZE); + msg->size = cea708_render(&cea708, sei_message_data(msg), + sei_message_size(msg)); + sei_message_append(&sei, msg); + } else if (output->caption_head) { + caption_frame_t cf; + caption_frame_init(&cf); + caption_frame_from_text(&cf, &output->caption_head->text[0]); + + sei_from_caption_frame(&sei, &cf); + + struct obs_caption_frame *next = output->caption_head->next; + bfree(output->caption_head); + output->caption_head = next; + } data = malloc(sei_render_size(&sei)); size = sei_render(&sei, data); @@ -1244,13 +1300,12 @@ static bool add_caption(struct obs_output *output, struct encoder_packet *out) sei_free(&sei); - struct caption_text *next = output->caption_head->next; - bfree(output->caption_head); - output->caption_head = next; return true; } #endif +double last_caption_timestamp = 0; + static inline void send_interleaved(struct obs_output *output) { struct encoder_packet out = output->interleaved_packets.array[0]; @@ -1286,6 +1341,13 @@ static inline void send_interleaved(struct obs_output *output) } } + if (output->caption_data.size > 0) { + if (last_caption_timestamp < frame_timestamp) { + last_caption_timestamp = frame_timestamp; + add_caption(output, &out); + } + } + pthread_mutex_unlock(&output->caption_mutex); #endif } @@ -2471,6 +2533,18 @@ const char *obs_output_get_id(const obs_output_t *output) : NULL; } +void obs_output_caption(obs_output_t *output, + const struct obs_source_cea_708 *captions) +{ + pthread_mutex_lock(&output->caption_mutex); + for (int i = 0; i < captions->packets; i++) { + circlebuf_push_back(&output->caption_data, + captions->data + (i * 3), + 3 * sizeof(uint8_t)); + } + pthread_mutex_unlock(&output->caption_mutex); +} + #if BUILD_CAPTIONS static struct caption_text *caption_text_new(const char *text, size_t bytes, struct caption_text *tail, diff --git a/libobs/obs-source.c b/libobs/obs-source.c index 41b2184bb..9aea45c73 100644 --- a/libobs/obs-source.c +++ b/libobs/obs-source.c @@ -184,6 +184,7 @@ static bool obs_source_init(struct obs_source *source) pthread_mutex_init_value(&source->audio_mutex); pthread_mutex_init_value(&source->audio_buf_mutex); pthread_mutex_init_value(&source->audio_cb_mutex); + pthread_mutex_init_value(&source->caption_cb_mutex); if (pthread_mutexattr_init(&attr) != 0) return false; @@ -201,6 +202,8 @@ static bool obs_source_init(struct obs_source *source) return false; if (pthread_mutex_init(&source->async_mutex, NULL) != 0) return false; + if (pthread_mutex_init(&source->caption_cb_mutex, NULL) != 0) + return false; if (is_audio_source(source) || is_composite_source(source)) allocate_audio_output_buffer(source); @@ -683,6 +686,7 @@ void obs_source_destroy(struct obs_source *source) da_free(source->audio_actions); da_free(source->audio_cb_list); + da_free(source->caption_cb_list); da_free(source->async_cache); da_free(source->async_frames); da_free(source->filters); @@ -691,6 +695,7 @@ void obs_source_destroy(struct obs_source *source) pthread_mutex_destroy(&source->audio_buf_mutex); pthread_mutex_destroy(&source->audio_cb_mutex); pthread_mutex_destroy(&source->audio_mutex); + pthread_mutex_destroy(&source->caption_cb_mutex); pthread_mutex_destroy(&source->async_mutex); obs_data_release(source->private_settings); obs_context_data_free(&source->context); @@ -2898,6 +2903,51 @@ void obs_source_set_async_rotation(obs_source_t *source, long rotation) source->async_rotation = rotation; } +void obs_source_output_cea708(obs_source_t *source, + const struct obs_source_cea_708 *captions) +{ + if (!captions) { + return; + } + + pthread_mutex_lock(&source->caption_cb_mutex); + + for (size_t i = source->caption_cb_list.num; i > 0; i--) { + struct caption_cb_info info = + source->caption_cb_list.array[i - 1]; + info.callback(info.param, source, captions); + } + + pthread_mutex_unlock(&source->caption_cb_mutex); +} + +void obs_source_add_caption_callback(obs_source_t *source, + obs_source_caption_t callback, void *param) +{ + struct caption_cb_info info = {callback, param}; + + if (!obs_source_valid(source, "obs_source_add_caption_callback")) + return; + + pthread_mutex_lock(&source->caption_cb_mutex); + da_push_back(source->caption_cb_list, &info); + pthread_mutex_unlock(&source->caption_cb_mutex); +} + +void obs_source_remove_caption_callback(obs_source_t *source, + obs_source_caption_t callback, + void *param) +{ + struct caption_cb_info info = {callback, param}; + + if (!obs_source_valid(source, "obs_source_remove_caption_callback")) + return; + + pthread_mutex_lock(&source->caption_cb_mutex); + da_erase_item(source->caption_cb_list, &info); + pthread_mutex_unlock(&source->caption_cb_mutex); +} + static inline bool preload_frame_changed(obs_source_t *source, const struct obs_source_frame *in) { diff --git a/libobs/obs-source.h b/libobs/obs-source.h index 6484242ec..ec4194fc2 100644 --- a/libobs/obs-source.h +++ b/libobs/obs-source.h @@ -186,6 +186,11 @@ enum obs_media_state { */ #define OBS_SOURCE_CONTROLLABLE_MEDIA (1 << 13) +/** + * Source type provides cea708 data + */ +#define OBS_SOURCE_CEA_708 (1 << 14) + /** @} */ typedef void (*obs_source_enum_proc_t)(obs_source_t *parent, diff --git a/libobs/obs.h b/libobs/obs.h index c917d224e..738a09e55 100644 --- a/libobs/obs.h +++ b/libobs/obs.h @@ -212,6 +212,12 @@ struct obs_source_audio { uint64_t timestamp; }; +struct obs_source_cea_708 { + const uint8_t *data; + uint32_t packets; + uint64_t timestamp; +}; + /** * Source asynchronous video output structure. Used with * obs_source_output_video to output asynchronous video. Video is buffered as @@ -1117,6 +1123,16 @@ EXPORT void obs_source_add_audio_capture_callback( EXPORT void obs_source_remove_audio_capture_callback( obs_source_t *source, obs_source_audio_capture_t callback, void *param); +typedef void (*obs_source_caption_t)(void *param, obs_source_t *source, + const struct obs_source_cea_708 *captions); + +EXPORT void obs_source_add_caption_callback(obs_source_t *source, + obs_source_caption_t callback, + void *param); +EXPORT void obs_source_remove_caption_callback(obs_source_t *source, + obs_source_caption_t callback, + void *param); + enum obs_deinterlace_mode { OBS_DEINTERLACE_MODE_DISABLE, OBS_DEINTERLACE_MODE_DISCARD, @@ -1208,6 +1224,9 @@ EXPORT void obs_source_output_video2(obs_source_t *source, EXPORT void obs_source_set_async_rotation(obs_source_t *source, long rotation); +EXPORT void obs_source_output_cea708(obs_source_t *source, + const struct obs_source_cea_708 *captions); + /** * Preloads asynchronous video data to allow instantaneous playback * @@ -1884,12 +1903,16 @@ EXPORT uint32_t obs_output_get_height(const obs_output_t *output); EXPORT const char *obs_output_get_id(const obs_output_t *output); +EXPORT void obs_output_caption(obs_output_t *output, + const struct obs_source_cea_708 *captions); + #if BUILD_CAPTIONS EXPORT void obs_output_output_caption_text1(obs_output_t *output, const char *text); EXPORT void obs_output_output_caption_text2(obs_output_t *output, const char *text, double display_duration); + #endif EXPORT float obs_output_get_congestion(obs_output_t *output); diff --git a/libobs/util/bitstream.c b/libobs/util/bitstream.c new file mode 100644 index 000000000..801e59cd4 --- /dev/null +++ b/libobs/util/bitstream.c @@ -0,0 +1,52 @@ +#include "bitstream.h" + +#include +#include + +void bitstream_reader_init(struct bitstream_reader *r, uint8_t *data, + size_t len) +{ + memset(r, 0, sizeof(struct bitstream_reader)); + r->buf = data; + r->subPos = 0x80; + r->len = len; +} + +uint8_t bitstream_reader_read_bit(struct bitstream_reader *r) +{ + if (r->pos >= r->len) + return 0; + + uint8_t bit = (*(r->buf + r->pos) & r->subPos) == r->subPos ? 1 : 0; + + r->subPos >>= 0x1; + if (r->subPos == 0) { + r->subPos = 0x80; + r->pos++; + } + + return bit; +} + +uint8_t bitstream_reader_read_bits(struct bitstream_reader *r, int bits) +{ + uint8_t res = 0; + + for (int i = 1; i <= bits; i++) { + res <<= 1; + res |= bitstream_reader_read_bit(r); + } + + return res; +} + +uint8_t bitstream_reader_r8(struct bitstream_reader *r) +{ + return bitstream_reader_read_bits(r, 8); +} + +uint16_t bitstream_reader_r16(struct bitstream_reader *r) +{ + uint8_t b = bitstream_reader_read_bits(r, 8); + return ((uint16_t)b << 8) | bitstream_reader_read_bits(r, 8); +} diff --git a/libobs/util/bitstream.h b/libobs/util/bitstream.h new file mode 100644 index 000000000..9612c900c --- /dev/null +++ b/libobs/util/bitstream.h @@ -0,0 +1,29 @@ +#pragma once + +#include "c99defs.h" + +/* + * General programmable serialization functions. (A shared interface to + * various reading/writing to/from different inputs/outputs) + */ + +#ifdef __cplusplus +extern "C" { +#endif + +struct bitstream_reader { + uint8_t pos; + uint8_t subPos; + uint8_t *buf; + size_t len; +}; + +EXPORT void bitstream_reader_init(struct bitstream_reader *r, uint8_t *data, + size_t len); +EXPORT uint8_t bitstream_reader_read_bits(struct bitstream_reader *r, int bits); +EXPORT uint8_t bitstream_reader_r8(struct bitstream_reader *r); +EXPORT uint16_t bitstream_reader_r16(struct bitstream_reader *r); + +#ifdef __cplusplus +} +#endif diff --git a/plugins/decklink/OBSVideoFrame.cpp b/plugins/decklink/OBSVideoFrame.cpp new file mode 100644 index 000000000..db87dd9b2 --- /dev/null +++ b/plugins/decklink/OBSVideoFrame.cpp @@ -0,0 +1,72 @@ +#include "OBSVideoFrame.h" + +OBSVideoFrame::OBSVideoFrame(long width, long height) +{ + this->width = width; + this->height = height; + this->rowBytes = width * 2; + this->data = new unsigned char[width * height * 2 + 1]; +} + +HRESULT OBSVideoFrame::SetFlags(BMDFrameFlags newFlags) +{ + flags = newFlags; + return S_OK; +} + +HRESULT OBSVideoFrame::SetTimecode(BMDTimecodeFormat format, + IDeckLinkTimecode *timecode) +{ + return 0; +} + +HRESULT +OBSVideoFrame::SetTimecodeFromComponents(BMDTimecodeFormat format, + uint8_t hours, uint8_t minutes, + uint8_t seconds, uint8_t frames, + BMDTimecodeFlags flags) +{ + return 0; +} + +HRESULT OBSVideoFrame::SetAncillaryData(IDeckLinkVideoFrameAncillary *ancillary) +{ + return 0; +} + +HRESULT OBSVideoFrame::SetTimecodeUserBits(BMDTimecodeFormat format, + BMDTimecodeUserBits userBits) +{ + return 0; +} + +long OBSVideoFrame::GetWidth() +{ + return width; +} + +long OBSVideoFrame::GetHeight() +{ + return height; +} + +long OBSVideoFrame::GetRowBytes() +{ + return rowBytes; +} + +BMDPixelFormat OBSVideoFrame::GetPixelFormat() +{ + return pixelFormat; +} + +BMDFrameFlags OBSVideoFrame::GetFlags() +{ + return flags; +} + +HRESULT OBSVideoFrame::GetBytes(void **buffer) +{ + *buffer = this->data; + return S_OK; +} diff --git a/plugins/decklink/OBSVideoFrame.h b/plugins/decklink/OBSVideoFrame.h new file mode 100644 index 000000000..46b15aa14 --- /dev/null +++ b/plugins/decklink/OBSVideoFrame.h @@ -0,0 +1,70 @@ +#pragma once + +#include "platform.hpp" + +class OBSVideoFrame : public IDeckLinkMutableVideoFrame { +private: + BMDFrameFlags flags; + BMDPixelFormat pixelFormat = bmdFormat8BitYUV; + + long width; + long height; + long rowBytes; + + unsigned char *data; + +public: + OBSVideoFrame(long width, long height); + + HRESULT STDMETHODCALLTYPE SetFlags(BMDFrameFlags newFlags) override; + + HRESULT STDMETHODCALLTYPE SetTimecode( + BMDTimecodeFormat format, IDeckLinkTimecode *timecode) override; + + HRESULT STDMETHODCALLTYPE SetTimecodeFromComponents( + BMDTimecodeFormat format, uint8_t hours, uint8_t minutes, + uint8_t seconds, uint8_t frames, + BMDTimecodeFlags flags) override; + + HRESULT + STDMETHODCALLTYPE + SetAncillaryData(IDeckLinkVideoFrameAncillary *ancillary) override; + + HRESULT STDMETHODCALLTYPE + SetTimecodeUserBits(BMDTimecodeFormat format, + BMDTimecodeUserBits userBits) override; + + long STDMETHODCALLTYPE GetWidth() override; + + long STDMETHODCALLTYPE GetHeight() override; + + long STDMETHODCALLTYPE GetRowBytes() override; + + BMDPixelFormat STDMETHODCALLTYPE GetPixelFormat() override; + + BMDFrameFlags STDMETHODCALLTYPE GetFlags() override; + + HRESULT STDMETHODCALLTYPE GetBytes(void **buffer) override; + + //Dummy implementations of remaining virtual methods + virtual HRESULT STDMETHODCALLTYPE + GetTimecode(/* in */ BMDTimecodeFormat format, + /* out */ IDeckLinkTimecode **timecode) + { + return E_NOINTERFACE; + }; + virtual HRESULT STDMETHODCALLTYPE + GetAncillaryData(/* out */ IDeckLinkVideoFrameAncillary **ancillary) + { + return E_NOINTERFACE; + }; + + // IUnknown interface (dummy implementation) + virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID iid, + LPVOID *ppv) + { + return E_NOINTERFACE; + } + virtual ULONG STDMETHODCALLTYPE AddRef() { return 1; } + virtual ULONG STDMETHODCALLTYPE Release() { return 1; } +}; diff --git a/plugins/decklink/decklink-device-instance.cpp b/plugins/decklink/decklink-device-instance.cpp index f75cc7286..170b31f96 100644 --- a/plugins/decklink/decklink-device-instance.cpp +++ b/plugins/decklink/decklink-device-instance.cpp @@ -9,8 +9,14 @@ #include #include +#include #include +#include "OBSVideoFrame.h" + +#include +#include + static inline enum video_format ConvertPixelFormat(BMDPixelFormat format) { switch (format) { @@ -62,14 +68,23 @@ static inline audio_repack_mode_t ConvertRepackFormat(speaker_layout format, DeckLinkDeviceInstance::DeckLinkDeviceInstance(DecklinkBase *decklink_, DeckLinkDevice *device_) - : currentFrame(), currentPacket(), decklink(decklink_), device(device_) + : currentFrame(), + currentPacket(), + currentCaptions(), + decklink(decklink_), + device(device_) { currentPacket.samples_per_sec = 48000; currentPacket.speakers = SPEAKERS_STEREO; currentPacket.format = AUDIO_FORMAT_16BIT; } -DeckLinkDeviceInstance::~DeckLinkDeviceInstance() {} +DeckLinkDeviceInstance::~DeckLinkDeviceInstance() +{ + if (convertFrame) { + delete convertFrame; + } +} void DeckLinkDeviceInstance::HandleAudioPacket( IDeckLinkAudioInputPacket *audioPacket, const uint64_t timestamp) @@ -127,16 +142,47 @@ void DeckLinkDeviceInstance::HandleVideoFrame( if (videoFrame == nullptr) return; + IDeckLinkVideoFrameAncillaryPackets *packets; + + if (videoFrame->QueryInterface(IID_IDeckLinkVideoFrameAncillaryPackets, + (void **)&packets) == S_OK) { + IDeckLinkAncillaryPacketIterator *iterator; + packets->GetPacketIterator(&iterator); + + IDeckLinkAncillaryPacket *packet; + iterator->Next(&packet); + + if (packet) { + auto did = packet->GetDID(); + auto sdid = packet->GetSDID(); + + // Caption data + if (did == 0x61 & sdid == 0x01) { + this->HandleCaptionPacket(packet, timestamp); + } + + packet->Release(); + } + + iterator->Release(); + packets->Release(); + } + + IDeckLinkVideoConversion *frameConverter = + CreateVideoConversionInstance(); + + frameConverter->ConvertFrame(videoFrame, convertFrame); + void *bytes; - if (videoFrame->GetBytes(&bytes) != S_OK) { + if (convertFrame->GetBytes(&bytes) != S_OK) { LOG(LOG_WARNING, "Failed to get video frame data"); return; } currentFrame.data[0] = (uint8_t *)bytes; - currentFrame.linesize[0] = (uint32_t)videoFrame->GetRowBytes(); - currentFrame.width = (uint32_t)videoFrame->GetWidth(); - currentFrame.height = (uint32_t)videoFrame->GetHeight(); + currentFrame.linesize[0] = (uint32_t)convertFrame->GetRowBytes(); + currentFrame.width = (uint32_t)convertFrame->GetWidth(); + currentFrame.height = (uint32_t)convertFrame->GetHeight(); currentFrame.timestamp = timestamp; obs_source_output_video2( @@ -144,6 +190,86 @@ void DeckLinkDeviceInstance::HandleVideoFrame( ¤tFrame); } +void DeckLinkDeviceInstance::HandleCaptionPacket( + IDeckLinkAncillaryPacket *packet, const uint64_t timestamp) +{ + auto line = packet->GetLineNumber(); + + const void *data; + uint32_t size; + packet->GetBytes(bmdAncillaryPacketFormatUInt8, &data, &size); + + auto anc = (uint8_t *)data; + struct bitstream_reader reader; + bitstream_reader_init(&reader, anc, size); + + auto header1 = bitstream_reader_r8(&reader); + auto header2 = bitstream_reader_r8(&reader); + + uint8_t length = bitstream_reader_r8(&reader); + uint8_t frameRate = bitstream_reader_read_bits(&reader, 4); + //reserved + bitstream_reader_read_bits(&reader, 4); + + auto cdp_timecode_added = bitstream_reader_read_bits(&reader, 1); + auto cdp_data_block_added = bitstream_reader_read_bits(&reader, 1); + auto cdp_service_info_added = bitstream_reader_read_bits(&reader, 1); + auto cdp_service_info_start = bitstream_reader_read_bits(&reader, 1); + auto cdp_service_info_changed = bitstream_reader_read_bits(&reader, 1); + auto cdp_service_info_end = bitstream_reader_read_bits(&reader, 1); + auto cdp_contains_captions = bitstream_reader_read_bits(&reader, 1); + //reserved + bitstream_reader_read_bits(&reader, 1); + + auto cdp_counter = bitstream_reader_r8(&reader); + auto cdp_counter2 = bitstream_reader_r8(&reader); + + if (cdp_timecode_added) { + auto timecodeSectionID = bitstream_reader_r8(&reader); + //reserved + bitstream_reader_read_bits(&reader, 2); + bitstream_reader_read_bits(&reader, 2); + bitstream_reader_read_bits(&reader, 4); + // reserved + bitstream_reader_read_bits(&reader, 1); + bitstream_reader_read_bits(&reader, 3); + bitstream_reader_read_bits(&reader, 4); + bitstream_reader_read_bits(&reader, 1); + bitstream_reader_read_bits(&reader, 3); + bitstream_reader_read_bits(&reader, 4); + bitstream_reader_read_bits(&reader, 1); + bitstream_reader_read_bits(&reader, 1); + bitstream_reader_read_bits(&reader, 3); + bitstream_reader_read_bits(&reader, 4); + } + + if (cdp_contains_captions) { + auto cdp_data_section = bitstream_reader_r8(&reader); + + auto process_em_data_flag = + bitstream_reader_read_bits(&reader, 1); + auto process_cc_data_flag = + bitstream_reader_read_bits(&reader, 1); + auto additional_data_flag = + bitstream_reader_read_bits(&reader, 1); + + auto cc_count = bitstream_reader_read_bits(&reader, 5); + + auto *outData = + (uint8_t *)bzalloc(sizeof(uint8_t) * cc_count * 3); + memcpy(outData, anc + reader.pos, cc_count * 3); + + currentCaptions.data = outData; + currentCaptions.timestamp = timestamp; + currentCaptions.packets = cc_count; + + obs_source_output_cea708( + static_cast(decklink)->GetSource(), + ¤tCaptions); + bfree(outData); + } +} + void DeckLinkDeviceInstance::FinalizeStream() { input->SetCallback(nullptr); @@ -189,6 +315,11 @@ void DeckLinkDeviceInstance::SetupVideoFormat(DeckLinkDeviceMode *mode_) currentFrame.color_range_min, currentFrame.color_range_max); + if (convertFrame) { + delete convertFrame; + } + convertFrame = new OBSVideoFrame(mode_->GetWidth(), mode_->GetHeight()); + #ifdef LOG_SETUP_VIDEO_FORMAT LOG(LOG_INFO, "Setup video format: %s, %s, %s", pixelFormat == bmdFormat8BitYUV ? "YUV" : "RGB", @@ -250,7 +381,7 @@ bool DeckLinkDeviceInstance::StartCapture(DeckLinkDeviceMode *mode_, bool isauto = mode_->GetName() == "Auto"; if (isauto) { displayMode = bmdModeNTSC; - pixelFormat = bmdFormat8BitYUV; + pixelFormat = bmdFormat10BitYUV; flags = bmdVideoInputEnableFormatDetection; } else { displayMode = mode_->GetDisplayMode(); @@ -503,7 +634,7 @@ HRESULT STDMETHODCALLTYPE DeckLinkDeviceInstance::VideoInputFormatChanged( default: case bmdDetectedVideoInputYCbCr422: - pixelFormat = bmdFormat8BitYUV; + pixelFormat = bmdFormat10BitYUV; break; } } diff --git a/plugins/decklink/decklink-device-instance.hpp b/plugins/decklink/decklink-device-instance.hpp index d2982bc6e..e6c3d6920 100644 --- a/plugins/decklink/decklink-device-instance.hpp +++ b/plugins/decklink/decklink-device-instance.hpp @@ -6,6 +6,7 @@ #include #include "decklink-device.hpp" #include "../../libobs/media-io/video-scaler.h" +#include "OBSVideoFrame.h" class AudioRepacker; class DecklinkBase; @@ -14,6 +15,7 @@ class DeckLinkDeviceInstance : public IDeckLinkInputCallback { protected: struct obs_source_frame2 currentFrame; struct obs_source_audio currentPacket; + struct obs_source_cea_708 currentCaptions; DecklinkBase *decklink = nullptr; DeckLinkDevice *device = nullptr; DeckLinkDeviceMode *mode = nullptr; @@ -34,6 +36,7 @@ protected: speaker_layout channelFormat = SPEAKERS_STEREO; bool swap; + OBSVideoFrame *convertFrame = nullptr; IDeckLinkMutableVideoFrame *decklinkOutputFrame = nullptr; void FinalizeStream(); @@ -104,4 +107,6 @@ public: void DisplayVideoFrame(video_data *frame); void WriteAudio(audio_data *frames); + void HandleCaptionPacket(IDeckLinkAncillaryPacket *packet, + const uint64_t timestamp); }; diff --git a/plugins/decklink/decklink-source.cpp b/plugins/decklink/decklink-source.cpp index 0b31623c6..1b3184476 100644 --- a/plugins/decklink/decklink-source.cpp +++ b/plugins/decklink/decklink-source.cpp @@ -331,9 +331,9 @@ struct obs_source_info create_decklink_source_info() struct obs_source_info decklink_source_info = {}; decklink_source_info.id = "decklink-input"; decklink_source_info.type = OBS_SOURCE_TYPE_INPUT; - decklink_source_info.output_flags = OBS_SOURCE_ASYNC_VIDEO | - OBS_SOURCE_AUDIO | - OBS_SOURCE_DO_NOT_DUPLICATE; + decklink_source_info.output_flags = + OBS_SOURCE_ASYNC_VIDEO | OBS_SOURCE_AUDIO | + OBS_SOURCE_DO_NOT_DUPLICATE | OBS_SOURCE_CEA_708; decklink_source_info.create = decklink_create; decklink_source_info.destroy = decklink_destroy; decklink_source_info.get_defaults = decklink_get_defaults; diff --git a/plugins/decklink/linux/CMakeLists.txt b/plugins/decklink/linux/CMakeLists.txt index 5c11bf056..b0b2d9a54 100644 --- a/plugins/decklink/linux/CMakeLists.txt +++ b/plugins/decklink/linux/CMakeLists.txt @@ -5,6 +5,8 @@ if(DISABLE_DECKLINK) return() endif() +include_directories(${CMAKE_SOURCE_DIR}/deps/libcaption) + set(linux-decklink-sdk_HEADERS decklink-sdk/DeckLinkAPI.h decklink-sdk/DeckLinkAPIConfiguration.h @@ -34,6 +36,7 @@ set(linux-decklink_HEADERS ../audio-repack.h ../audio-repack.hpp ../util.hpp + ../OBSVideoFrame.h ) set(linux-decklink_SOURCES @@ -51,6 +54,7 @@ set(linux-decklink_SOURCES ../audio-repack.c platform.cpp ../util.cpp + ../OBSVideoFrame.h ) add_library(linux-decklink MODULE @@ -62,7 +66,7 @@ add_library(linux-decklink MODULE target_link_libraries(linux-decklink libobs - ) + caption) set_target_properties(linux-decklink PROPERTIES FOLDER "plugins") install_obs_plugin_with_data(linux-decklink ../data) diff --git a/plugins/decklink/mac/CMakeLists.txt b/plugins/decklink/mac/CMakeLists.txt index adfd59278..eea45244b 100644 --- a/plugins/decklink/mac/CMakeLists.txt +++ b/plugins/decklink/mac/CMakeLists.txt @@ -9,6 +9,8 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}") find_library(COREFOUNDATION CoreFoundation) +include_directories(${CMAKE_SOURCE_DIR}/deps/libcaption) + set(mac-decklink-sdk_HEADERS decklink-sdk/DeckLinkAPI.h decklink-sdk/DeckLinkAPIConfiguration.h @@ -37,6 +39,7 @@ set(mac-decklink_HEADERS ../audio-repack.h ../audio-repack.hpp ../util.hpp + ../OBSVideoFrame.h ) set(mac-decklink_SOURCES @@ -54,6 +57,7 @@ set(mac-decklink_SOURCES ../audio-repack.c platform.cpp ../util.cpp + ../OBSVideoFrame.cpp ) list(APPEND decklink_HEADERS ${decklink_UI_HEADERS}) @@ -73,7 +77,9 @@ add_library(mac-decklink MODULE target_link_libraries(mac-decklink libobs - ${COREFOUNDATION}) + obs-frontend-api + ${COREFOUNDATION} + caption) set_target_properties(mac-decklink PROPERTIES FOLDER "plugins") install_obs_plugin_with_data(mac-decklink ../data) diff --git a/plugins/decklink/platform.hpp b/plugins/decklink/platform.hpp index 80b25a7ec..21820d556 100644 --- a/plugins/decklink/platform.hpp +++ b/plugins/decklink/platform.hpp @@ -7,6 +7,7 @@ typedef BOOL decklink_bool_t; typedef BSTR decklink_string_t; IDeckLinkDiscovery *CreateDeckLinkDiscoveryInstance(void); IDeckLinkIterator *CreateDeckLinkIteratorInstance(void); +IDeckLinkVideoConversion *CreateVideoConversionInstance(void); #define IUnknownUUID IID_IUnknown typedef REFIID CFUUIDBytes; #define CFUUIDGetUUIDBytes(x) x diff --git a/plugins/decklink/win/CMakeLists.txt b/plugins/decklink/win/CMakeLists.txt index fca4e5dc3..270108712 100644 --- a/plugins/decklink/win/CMakeLists.txt +++ b/plugins/decklink/win/CMakeLists.txt @@ -7,6 +7,8 @@ endif() include(IDLFileHelper) +include_directories(${CMAKE_SOURCE_DIR}/deps/libcaption) + set(win-decklink-sdk_IDLS decklink-sdk/DeckLinkAPI.idl ) @@ -29,6 +31,7 @@ set(win-decklink_HEADERS ../audio-repack.h ../audio-repack.hpp ../util.hpp + ../OBSVideoFrame.h ) set(MODULE_DESCRIPTION "OBS DeckLink Windows module") @@ -48,7 +51,8 @@ set(win-decklink_SOURCES ../audio-repack.c platform.cpp ../util.cpp - win-decklink.rc) + win-decklink.rc + ../OBSVideoFrame.cpp) add_idl_files(win-decklink-sdk_GENERATED_FILES ${win-decklink-sdk_IDLS} @@ -56,6 +60,7 @@ add_idl_files(win-decklink-sdk_GENERATED_FILES include_directories( ${CMAKE_CURRENT_BINARY_DIR} + "${CMAKE_SOURCE_DIR}/UI/obs-frontend-api" ) add_library(win-decklink MODULE @@ -66,7 +71,9 @@ add_library(win-decklink MODULE ) target_link_libraries(win-decklink - libobs) + libobs + obs-frontend-api + caption) set_target_properties(win-decklink PROPERTIES FOLDER "plugins") install_obs_plugin_with_data(win-decklink ../data) diff --git a/plugins/decklink/win/platform.cpp b/plugins/decklink/win/platform.cpp index 260797d01..59e0721f0 100644 --- a/plugins/decklink/win/platform.cpp +++ b/plugins/decklink/win/platform.cpp @@ -20,6 +20,16 @@ IDeckLinkIterator *CreateDeckLinkIteratorInstance(void) return result == S_OK ? iterator : nullptr; } +IDeckLinkVideoConversion *CreateVideoConversionInstance(void) +{ + IDeckLinkVideoConversion *conversion; + const HRESULT result = CoCreateInstance(CLSID_CDeckLinkVideoConversion, + nullptr, CLSCTX_ALL, + IID_IDeckLinkVideoConversion, + (void **)&conversion); + return result == S_OK ? conversion : nullptr; +} + bool DeckLinkStringToStdString(decklink_string_t input, std::string &output) { if (input == nullptr) diff --git a/test/cmocka/CMakeLists.txt b/test/cmocka/CMakeLists.txt index 747d4321b..e1a5bb710 100644 --- a/test/cmocka/CMakeLists.txt +++ b/test/cmocka/CMakeLists.txt @@ -37,3 +37,10 @@ target_link_libraries(test_darray ${CMOCKA_LIBRARIES} libobs) add_test(test_darray ${CMAKE_CURRENT_BINARY_DIR}/test_darray) fixLink(test_darray) + +# bitstream test +add_executable(test_bitstream test_bitstream.c) +target_link_libraries(test_bitstream ${CMOCKA_LIBRARIES} libobs) + +add_test(test_bitstream ${CMAKE_CURRENT_BINARY_DIR}/test_bitstream) +fixLink(test_bitstream) diff --git a/test/cmocka/test_bitstream.c b/test/cmocka/test_bitstream.c new file mode 100644 index 000000000..8bc026281 --- /dev/null +++ b/test/cmocka/test_bitstream.c @@ -0,0 +1,34 @@ +#include +#include +#include +#include + +#include + +static void bitstream_test(void **state) +{ + struct bitstream_reader reader; + uint8_t data[6] = {0x34, 0xff, 0xe1, 0x23, 0x91, 0x45}; + + // set len to one less than the array to show that we stop reading at that len + bitstream_reader_init(&reader, data, 5); + + assert_int_equal(bitstream_reader_read_bits(&reader, 8), 0x34); + assert_int_equal(bitstream_reader_read_bits(&reader, 1), 1); + assert_int_equal(bitstream_reader_read_bits(&reader, 3), 7); + assert_int_equal(bitstream_reader_read_bits(&reader, 4), 0xF); + assert_int_equal(bitstream_reader_r8(&reader), 0xe1); + assert_int_equal(bitstream_reader_r16(&reader), 0x2391); + + // test reached end + assert_int_equal(bitstream_reader_r8(&reader), 0); +} + +int main() +{ + const struct CMUnitTest tests[] = { + cmocka_unit_test(bitstream_test), + }; + + return cmocka_run_group_tests(tests, NULL, NULL); +}