Files
obs-studio/shared/bpm/bpm.c
2025-12-17 18:02:19 -05:00

599 lines
21 KiB
C

#include "obs.h"
#include "bpm-internal.h"
static void render_metrics_time(struct metrics_time *m_time)
{
/* Generate the RFC3339 time string from the timespec struct, for example:
*
* "2024-05-31T12:26:03.591Z"
*/
memset(&m_time->rfc3339_str, 0, sizeof(m_time->rfc3339_str));
strftime(m_time->rfc3339_str, sizeof(m_time->rfc3339_str), "%Y-%m-%dT%T", gmtime(&m_time->tspec.tv_sec));
sprintf(m_time->rfc3339_str + strlen(m_time->rfc3339_str), ".%03ldZ", m_time->tspec.tv_nsec / 1000000);
m_time->valid = true;
}
static bool update_metrics(obs_output_t *output, const struct encoder_packet *pkt,
const struct encoder_packet_time *ept, struct metrics_data *m_track)
{
if (!pkt) {
blog(LOG_DEBUG, "%s: Null encoder_packet pointer", __FUNCTION__);
return false;
}
if (!output || !ept || !m_track) {
blog(LOG_DEBUG, "%s: Null arguments for track %lu", __FUNCTION__, pkt->track_idx);
return false;
}
// Perform reads on all the counters as close together as possible
m_track->session_frames_output.curr = obs_output_get_total_frames(output);
m_track->session_frames_dropped.curr = obs_output_get_frames_dropped(output);
m_track->session_frames_rendered.curr = obs_get_total_frames();
m_track->session_frames_lagged.curr = obs_get_lagged_frames();
const video_t *video = obs_encoder_video(pkt->encoder);
if (video) {
/* video_output_get_total_frames() returns the number of frames
* before the framerate decimator. For example, if the OBS session
* is rendering at 60fps, and the rendition is set for 30 fps,
* the counter will increment by 60 per second, not 30 per second.
* For metrics we will consider this value to be the number of
* frames input to the obs_encoder_t instance.
*/
m_track->rendition_frames_input.curr = video_output_get_total_frames(video);
m_track->rendition_frames_skipped.curr = video_output_get_skipped_frames(video);
/* obs_encoder_get_encoded_frames() returns the number of frames
* successfully encoded by the obs_encoder_t instance.
*/
m_track->rendition_frames_output.curr = obs_encoder_get_encoded_frames(pkt->encoder);
} else {
m_track->rendition_frames_input.curr = 0;
m_track->rendition_frames_skipped.curr = 0;
m_track->rendition_frames_output.curr = 0;
blog(LOG_ERROR, "update_metrics(): *video_t==null");
}
// Set the diff values to 0 if PTS is 0
if (pkt->pts == 0) {
m_track->session_frames_output.diff = 0;
m_track->session_frames_dropped.diff = 0;
m_track->session_frames_rendered.diff = 0;
m_track->session_frames_lagged.diff = 0;
m_track->rendition_frames_input.diff = 0;
m_track->rendition_frames_skipped.diff = 0;
m_track->rendition_frames_output.diff = 0;
blog(LOG_DEBUG, "update_metrics(): Setting diffs to 0");
} else {
// Calculate diff's
m_track->session_frames_output.diff =
m_track->session_frames_output.curr - m_track->session_frames_output.ref;
m_track->session_frames_dropped.diff =
m_track->session_frames_dropped.curr - m_track->session_frames_dropped.ref;
m_track->session_frames_rendered.diff =
m_track->session_frames_rendered.curr - m_track->session_frames_rendered.ref;
m_track->session_frames_lagged.diff =
m_track->session_frames_lagged.curr - m_track->session_frames_lagged.ref;
m_track->rendition_frames_input.diff =
m_track->rendition_frames_input.curr - m_track->rendition_frames_input.ref;
m_track->rendition_frames_skipped.diff =
m_track->rendition_frames_skipped.curr - m_track->rendition_frames_skipped.ref;
m_track->rendition_frames_output.diff =
m_track->rendition_frames_output.curr - m_track->rendition_frames_output.ref;
}
// Update the reference values
m_track->session_frames_output.ref = m_track->session_frames_output.curr;
m_track->session_frames_dropped.ref = m_track->session_frames_dropped.curr;
m_track->session_frames_rendered.ref = m_track->session_frames_rendered.curr;
m_track->session_frames_lagged.ref = m_track->session_frames_lagged.curr;
m_track->rendition_frames_input.ref = m_track->rendition_frames_input.curr;
m_track->rendition_frames_skipped.ref = m_track->rendition_frames_skipped.curr;
m_track->rendition_frames_output.ref = m_track->rendition_frames_output.curr;
/* BPM Timestamp Message */
m_track->cts.valid = false;
m_track->ferts.valid = false;
m_track->fercts.valid = false;
/* Generate the timestamp representations for CTS, FER, and FERC.
* Check if each is non-zero and that temporal consistency is correct:
* FEC > FERC > CTS
* FEC and FERC depends on CTS, and FERC depends on FER, so ensure
* we only signal an integral set of timestamps.
*/
os_nstime_to_timespec(ept->cts, &m_track->cts.tspec);
render_metrics_time(&m_track->cts);
if (ept->fer && (ept->fer > ept->cts)) {
os_nstime_to_timespec(ept->fer, &m_track->ferts.tspec);
render_metrics_time(&m_track->ferts);
if (ept->ferc && (ept->ferc > ept->fer)) {
os_nstime_to_timespec(ept->ferc, &m_track->fercts.tspec);
render_metrics_time(&m_track->fercts);
}
}
// Always generate the timestamp representation for PIR
m_track->pirts.valid = false;
os_nstime_to_timespec(ept->pir, &m_track->pirts.tspec);
render_metrics_time(&m_track->pirts);
/* Log the BPM timestamp and frame counter information. This
* provides visibility into the metrics when OBS is started
* with "--verbose" and "--unfiltered_log".
*/
blog(LOG_DEBUG,
"BPM: %s, trk %lu: [CTS|FER-CTS|FERC-FER|PIR-CTS]:[%" PRIu64 " ms|%" PRIu64 " ms|%" PRIu64 " us|%" PRIu64
" ms], [dts|pts]:[%" PRId64 "|%" PRId64 "], S[R:O:D:L],R[I:S:O]:%d:%d:%d:%d:%d:%d:%d",
obs_encoder_get_name(pkt->encoder), pkt->track_idx, ept->cts / 1000000, (ept->fer - ept->cts) / 1000000,
(ept->ferc - ept->fer) / 1000, (ept->pir - ept->cts) / 1000000, pkt->dts, pkt->pts,
m_track->session_frames_rendered.diff, m_track->session_frames_output.diff,
m_track->session_frames_dropped.diff, m_track->session_frames_lagged.diff,
m_track->rendition_frames_input.diff, m_track->rendition_frames_skipped.diff,
m_track->rendition_frames_output.diff);
return true;
}
void bpm_ts_sei_render(struct metrics_data *m_track)
{
uint8_t num_timestamps = 0;
struct serializer s;
m_track->sei_rendered[BPM_TS_SEI] = false;
// Initialize the output array here; caller is responsible to free it
array_output_serializer_init(&s, &m_track->sei_payload[BPM_TS_SEI]);
// Write the UUID for this SEI message
s_write(&s, bpm_ts_uuid, sizeof(bpm_ts_uuid));
// Determine how many timestamps are valid
if (m_track->cts.valid)
num_timestamps++;
if (m_track->ferts.valid)
num_timestamps++;
if (m_track->fercts.valid)
num_timestamps++;
if (m_track->pirts.valid)
num_timestamps++;
/* Encode number of timestamps for this SEI. Upper 4 bits are
* set to b0000 (reserved); lower 4-bits num_timestamps - 1.
*/
s_w8(&s, (num_timestamps - 1) & 0x0F);
if (m_track->cts.valid) {
// Timestamp type
s_w8(&s, BPM_TS_RFC3339);
// Write the timestamp event tag (Composition Time Event)
s_w8(&s, BPM_TS_EVENT_CTS);
// Write the RFC3339-formatted string, including the null terminator
s_write(&s, m_track->cts.rfc3339_str, strlen(m_track->cts.rfc3339_str) + 1);
}
if (m_track->ferts.valid) {
// Timestamp type
s_w8(&s, BPM_TS_RFC3339);
// Write the timestamp event tag (Frame Encode Request Event)
s_w8(&s, BPM_TS_EVENT_FER);
// Write the RFC3339-formatted string, including the null terminator
s_write(&s, m_track->ferts.rfc3339_str, strlen(m_track->ferts.rfc3339_str) + 1);
}
if (m_track->fercts.valid) {
// Timestamp type
s_w8(&s, BPM_TS_RFC3339);
// Write the timestamp event tag (Frame Encode Request Complete Event)
s_w8(&s, BPM_TS_EVENT_FERC);
// Write the RFC3339-formatted string, including the null terminator
s_write(&s, m_track->fercts.rfc3339_str, strlen(m_track->fercts.rfc3339_str) + 1);
}
if (m_track->pirts.valid) {
// Timestamp type
s_w8(&s, BPM_TS_RFC3339);
// Write the timestamp event tag (Packet Interleave Request Event)
s_w8(&s, BPM_TS_EVENT_PIR);
// Write the RFC3339-formatted string, including the null terminator
s_write(&s, m_track->pirts.rfc3339_str, strlen(m_track->pirts.rfc3339_str) + 1);
}
m_track->sei_rendered[BPM_TS_SEI] = true;
}
void bpm_sm_sei_render(struct metrics_data *m_track)
{
uint8_t num_timestamps = 0;
uint8_t num_counters = 0;
struct serializer s;
m_track->sei_rendered[BPM_SM_SEI] = false;
// Initialize the output array here; caller is responsible to free it
array_output_serializer_init(&s, &m_track->sei_payload[BPM_SM_SEI]);
// Write the UUID for this SEI message
s_write(&s, bpm_sm_uuid, sizeof(bpm_sm_uuid));
// Encode number of timestamps for this SEI
num_timestamps = 1;
// Upper 4 bits are set to b0000 (reserved); lower 4-bits num_timestamps - 1
s_w8(&s, (num_timestamps - 1) & 0x0F);
// Timestamp type
s_w8(&s, BPM_TS_RFC3339);
/* Write the timestamp event tag (Packet Interleave Request Event).
* Use the PIR_TS timestamp because the data was all collected at that time.
*/
s_w8(&s, BPM_TS_EVENT_PIR);
// Write the RFC3339-formatted string, including the null terminator
s_write(&s, m_track->pirts.rfc3339_str, strlen(m_track->pirts.rfc3339_str) + 1);
// Session metrics has 4 counters
num_counters = 4;
/* Send all the counters with a tag(8-bit):value(32-bit) configuration.
* Upper 4 bits are set to b0000 (reserved); lower 4-bits num_counters - 1.
*/
s_w8(&s, (num_counters - 1) & 0x0F);
s_w8(&s, BPM_SM_FRAMES_RENDERED);
s_wb32(&s, m_track->session_frames_rendered.diff);
s_w8(&s, BPM_SM_FRAMES_LAGGED);
s_wb32(&s, m_track->session_frames_lagged.diff);
s_w8(&s, BPM_SM_FRAMES_DROPPED);
s_wb32(&s, m_track->session_frames_dropped.diff);
s_w8(&s, BPM_SM_FRAMES_OUTPUT);
s_wb32(&s, m_track->session_frames_output.diff);
m_track->sei_rendered[BPM_SM_SEI] = true;
}
void bpm_erm_sei_render(struct metrics_data *m_track)
{
uint8_t num_timestamps = 0;
uint8_t num_counters = 0;
struct serializer s;
m_track->sei_rendered[BPM_ERM_SEI] = false;
// Initialize the output array here; caller is responsible to free it
array_output_serializer_init(&s, &m_track->sei_payload[BPM_ERM_SEI]);
// Write the UUID for this SEI message
s_write(&s, bpm_erm_uuid, sizeof(bpm_erm_uuid));
// Encode number of timestamps for this SEI
num_timestamps = 1;
// Upper 4 bits are set to b0000 (reserved); lower 4-bits num_timestamps - 1
s_w8(&s, (num_timestamps - 1) & 0x0F);
// Timestamp type
s_w8(&s, BPM_TS_RFC3339);
/* Write the timestamp event tag (Packet Interleave Request Event).
* Use the PIRTS timestamp because the data was all collected at that time.
*/
s_w8(&s, BPM_TS_EVENT_PIR);
// Write the RFC3339-formatted string, including the null terminator
s_write(&s, m_track->pirts.rfc3339_str, strlen(m_track->pirts.rfc3339_str) + 1);
// Encoder rendition metrics has 3 counters
num_counters = 3;
/* Send all the counters with a tag(8-bit):value(32-bit) configuration.
* Upper 4 bits are set to b0000 (reserved); lower 4-bits num_counters - 1.
*/
s_w8(&s, (num_counters - 1) & 0x0F);
s_w8(&s, BPM_ERM_FRAMES_INPUT);
s_wb32(&s, m_track->rendition_frames_input.diff);
s_w8(&s, BPM_ERM_FRAMES_SKIPPED);
s_wb32(&s, m_track->rendition_frames_skipped.diff);
s_w8(&s, BPM_ERM_FRAMES_OUTPUT);
s_wb32(&s, m_track->rendition_frames_output.diff);
m_track->sei_rendered[BPM_ERM_SEI] = true;
}
/* Note : extract_buffer_from_sei() and nal_start are also defined
* in obs-output.c, however they are not public APIs. When the caption
* library is re-worked, this code should be refactored into that.
*/
static size_t extract_buffer_from_sei(sei_t *sei, uint8_t **data_out)
{
if (!sei || !sei->head) {
return 0;
}
/* We should only need to get one payload, because the SEI that was
* generated should only have one message, so no need to iterate. If
* we did iterate, we would need to generate multiple OBUs. */
sei_message_t *msg = sei_message_head(sei);
int payload_size = (int)sei_message_size(msg);
uint8_t *payload_data = sei_message_data(msg);
*data_out = bmalloc(payload_size);
memcpy(*data_out, payload_data, payload_size);
return payload_size;
}
static const uint8_t nal_start[4] = {0, 0, 0, 1};
/* process_metrics() will update and insert unregistered
* SEI (AVC/HEVC) or OBU (AV1) messages into the encoded
* video bitstream.
*/
static bool process_metrics(obs_output_t *output, struct encoder_packet *out, struct encoder_packet_time *ept,
struct metrics_data *m_track)
{
struct encoder_packet backup = *out;
sei_t sei;
uint8_t *data = NULL;
size_t size;
long ref = 1;
bool avc = false;
bool hevc = false;
bool av1 = false;
if (!m_track) {
blog(LOG_DEBUG, "Metrics track for index: %lu had not be initialized", out->track_idx);
return false;
}
// Update the metrics for this track
if (!update_metrics(output, out, ept, m_track)) {
// Something went wrong; log it and return
blog(LOG_DEBUG, "update_metrics() for track index: %lu failed", out->track_idx);
return false;
}
if (strcmp(obs_encoder_get_codec(out->encoder), "h264") == 0) {
avc = true;
} else if (strcmp(obs_encoder_get_codec(out->encoder), "av1") == 0) {
av1 = true;
#ifdef ENABLE_HEVC
} else if (strcmp(obs_encoder_get_codec(out->encoder), "hevc") == 0) {
hevc = true;
#endif
}
#ifdef ENABLE_HEVC
uint8_t hevc_nal_header[2];
if (hevc) {
size_t nal_header_index_start = 4;
// Skip past the annex-b start code
if (memcmp(out->data, nal_start + 1, 3) == 0) {
nal_header_index_start = 3;
} else if (memcmp(out->data, nal_start, 4) == 0) {
nal_header_index_start = 4;
} else {
/* We shouldn't ever see this unless we start getting
* packets without annex-b start codes. */
blog(LOG_DEBUG, "Annex-B start code not found, we may not "
"generate a valid hevc nal unit header "
"for our caption");
return false;
}
/* We will use the same 2 byte NAL unit header for the SEI,
* but swap the NAL types out. */
hevc_nal_header[0] = out->data[nal_header_index_start];
hevc_nal_header[1] = out->data[nal_header_index_start + 1];
}
#endif
// Create array for the original packet data + the SEI appended data
DARRAY(uint8_t) out_data;
da_init(out_data);
// Copy the original packet
da_push_back_array(out_data, (uint8_t *)&ref, sizeof(ref));
da_push_back_array(out_data, out->data, out->size);
// Build the SEI metrics message payload
bpm_ts_sei_render(m_track);
bpm_sm_sei_render(m_track);
bpm_erm_sei_render(m_track);
// Iterate over all the BPM SEI types
for (uint8_t i = 0; i < BPM_MAX_SEI; ++i) {
// Create and inject the syntax specific SEI messages in the bitstream if the rendering was successful
if (m_track->sei_rendered[i]) {
// Send one SEI message per NALU or OBU
sei_init(&sei, 0.0);
// Generate the formatted SEI message
sei_message_t *msg = sei_message_new(sei_type_user_data_unregistered,
m_track->sei_payload[i].bytes.array,
m_track->sei_payload[i].bytes.num);
sei_message_append(&sei, msg);
// Free the SEI payload buffer in the metrics track
array_output_serializer_free(&m_track->sei_payload[i]);
// Update for any codec specific syntax and add to the output bitstream
if (avc || hevc || av1) {
if (avc || hevc) {
data = bmalloc(sei_render_size(&sei));
size = sei_render(&sei, data);
}
/* In each of these specs there is an identical structure that
* carries user private metadata. We have an AVC SEI wrapped
* version of that here. We will strip it out and repackage
* it slightly to fit the different codec carrying mechanisms.
* A slightly modified SEI for HEVC and a metadata OBU for AV1.
*/
if (avc) {
/* TODO: SEI should come after AUD/SPS/PPS,
* but before any VCL */
da_push_back_array(out_data, nal_start, 4);
da_push_back_array(out_data, data, size);
#ifdef ENABLE_HEVC
} else if (hevc) {
/* Only first NAL (VPS/PPS/SPS) should use the 4 byte
* start code. SEIs use 3 byte version */
da_push_back_array(out_data, nal_start + 1, 3);
/* nal_unit_header( ) {
* forbidden_zero_bit f(1)
* nal_unit_type u(6)
* nuh_layer_id u(6)
* nuh_temporal_id_plus1 u(3)
* }
*/
const uint8_t suffix_sei_nal_type = 40;
/* The first bit is always 0, so we just need to
* save the last bit off the original header and
* add the SEI NAL type. */
uint8_t first_byte = (suffix_sei_nal_type << 1) | (0x01 & hevc_nal_header[0]);
hevc_nal_header[0] = first_byte;
/* The HEVC NAL unit header is 2 byte instead of
* one, otherwise everything else is the
* same. */
da_push_back_array(out_data, hevc_nal_header, 2);
da_push_back_array(out_data, &data[1], size - 1);
#endif
} else if (av1) {
uint8_t *obu_buffer = NULL;
size_t obu_buffer_size = 0;
size = extract_buffer_from_sei(&sei, &data);
metadata_obu(data, size, &obu_buffer, &obu_buffer_size,
METADATA_TYPE_USER_PRIVATE_6);
if (obu_buffer) {
da_push_back_array(out_data, obu_buffer, obu_buffer_size);
bfree(obu_buffer);
}
}
if (data) {
bfree(data);
}
}
sei_free(&sei);
}
}
obs_encoder_packet_release(out);
*out = backup;
out->data = (uint8_t *)out_data.array + sizeof(ref);
out->size = out_data.num - sizeof(ref);
if (avc || hevc || av1) {
return true;
}
return false;
}
static struct metrics_data *bpm_create_metrics_track(void)
{
struct metrics_data *rval = bzalloc(sizeof(struct metrics_data));
pthread_mutex_init_value(&rval->metrics_mutex);
if (pthread_mutex_init(&rval->metrics_mutex, NULL) != 0) {
bfree(rval);
rval = NULL;
}
return rval;
}
static bool bpm_get_track(obs_output_t *output, size_t track, struct metrics_data **m_track)
{
bool found = false;
// Walk the DARRAY looking for the output pointer
pthread_mutex_lock(&bpm_metrics_mutex);
for (size_t i = bpm_metrics.num; i > 0; i--) {
if (output == bpm_metrics.array[i - 1].output) {
*m_track = bpm_metrics.array[i - 1].metrics_tracks[track];
found = true;
break;
}
}
if (!found) {
// Create the new BPM metrics entries
struct output_metrics_link *oml = da_push_back_new(bpm_metrics);
oml->output = output;
for (size_t i = 0; i < MAX_OUTPUT_VIDEO_ENCODERS; ++i) {
oml->metrics_tracks[i] = bpm_create_metrics_track();
}
*m_track = oml->metrics_tracks[track];
found = true;
}
pthread_mutex_unlock(&bpm_metrics_mutex);
return found;
}
static void bpm_init(void)
{
pthread_mutex_init_value(&bpm_metrics_mutex);
da_init(bpm_metrics);
}
void bpm_destroy(obs_output_t *output)
{
int64_t idx = -1;
pthread_once(&bpm_once, bpm_init);
pthread_mutex_lock(&bpm_metrics_mutex);
// Walk the DARRAY looking for the index that matches the output
for (size_t i = bpm_metrics.num; i > 0; i--) {
if (output == bpm_metrics.array[i - 1].output) {
idx = i - 1;
break;
}
}
if (idx >= 0) {
struct output_metrics_link *oml = &bpm_metrics.array[idx];
for (size_t i = 0; i < MAX_OUTPUT_VIDEO_ENCODERS; i++) {
if (oml->metrics_tracks[i]) {
struct metrics_data *m_track = oml->metrics_tracks[i];
for (uint8_t j = 0; j < BPM_MAX_SEI; ++j) {
array_output_serializer_free(&m_track->sei_payload[j]);
}
pthread_mutex_destroy(&m_track->metrics_mutex);
bfree(m_track);
m_track = NULL;
}
}
da_erase(bpm_metrics, idx);
if (bpm_metrics.num == 0)
da_free(bpm_metrics);
}
pthread_mutex_unlock(&bpm_metrics_mutex);
}
/* bpm_inject() is the callback function that needs to be registered
* with each output needing Broadcast Performance Metrics injected
* into the video bitstream, using SEI (AVC/HEVC) and OBU (AV1) syntax.
*/
void bpm_inject(obs_output_t *output, struct encoder_packet *pkt, struct encoder_packet_time *pkt_time, void *param)
{
UNUSED_PARAMETER(param);
pthread_once(&bpm_once, bpm_init);
if (!output || !pkt) {
blog(LOG_DEBUG, "%s: Null pointer arguments supplied, returning", __FUNCTION__);
return;
}
/* Insert BPM on video frames and only when a keyframe
* is detected.
*/
if (pkt->type == OBS_ENCODER_VIDEO && pkt->keyframe) {
/* Video packet must have pkt_timing supplied for BPM */
if (!pkt_time) {
blog(LOG_DEBUG, "%s: Packet timing missing for track %ld, PTS %" PRId64, __FUNCTION__,
pkt->track_idx, pkt->pts);
return;
}
/* Get the metrics track associated with the output.
* Allocate BPM metrics structures for the output if needed.
*/
struct metrics_data *m_track = NULL;
if (!bpm_get_track(output, pkt->track_idx, &m_track)) {
blog(LOG_DEBUG, "%s: BPM metrics track not found!", __FUNCTION__);
return;
}
pthread_mutex_lock(&m_track->metrics_mutex);
/* Update the metrics and generate BPM messages. */
if (!process_metrics(output, pkt, pkt_time, m_track)) {
blog(LOG_DEBUG, "%s: BPM injection processing failed", __FUNCTION__);
}
pthread_mutex_unlock(&m_track->metrics_mutex);
}
}