// // ZoneMinder Ffmpeg Camera Class Implementation // Copyright (C) 2001-2008 Philip Coombes // // This program is free software; you can redistribute it and/or // modify it under the terms of the GNU General Public License // as published by the Free Software Foundation; either version 2 // of the License, or (at your option) any later version. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU General Public License // along with this program; if not, write to the Free Software // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. // #include "zm_ffmpeg_camera.h" #include "zm_ffmpeg_input.h" #include "zm_monitor.h" #include "zm_packet.h" #include "zm_signal.h" #include "zm_utils.h" #include "url.hpp" #include #include extern "C" { #include #include } TimePoint start_read_time; FfmpegCamera::FfmpegCamera( const Monitor *monitor, const std::string &p_path, const std::string &p_second_path, const std::string &p_user, const std::string &p_pass, const std::string &p_method, const std::string &p_options, int p_width, int p_height, int p_colours, int p_brightness, int p_contrast, int p_hue, int p_colour, bool p_capture, bool p_record_audio, const std::string &p_hwaccel_name, const std::string &p_hwaccel_device) : Camera( monitor, FFMPEG_SRC, p_width, p_height, p_colours, ZM_SUBPIX_ORDER_DEFAULT_FOR_COLOUR(p_colours), p_brightness, p_contrast, p_hue, p_colour, p_capture, p_record_audio ), mPath(p_path), mSecondPath(p_second_path), mUser(p_user), mPass(p_pass), mMethod(p_method), mOptions(p_options), hwaccel_name(p_hwaccel_name), hwaccel_device(p_hwaccel_device), mSecondInput(nullptr), frameCount(0), use_hwaccel(true), mConvertContext(nullptr), error_count(0), stream_width(0), stream_height(0) { mMaskedPath = remove_authentication(mPath); mMaskedSecondPath = remove_authentication(mSecondPath); mLoop = false; mLoopVideoOffset = 0; mLoopAudioOffset = 0; mLoopVideoFrameDuration = 0; mLoopAudioFrameDuration = 0; mRealtime = false; mRealtimeAnchored = false; mRealtimeStartTS = 0; if ( capture ) { FFMPEGInit(); } #if HAVE_LIBAVUTIL_HWCONTEXT_H hw_device_ctx = nullptr; hw_pix_fmt = AV_PIX_FMT_NONE; #endif /* Has to be located inside the constructor so other components such as zma * will receive correct colours and subpixel order */ if ( zm_is_rgb32(pixelFormat) ) { subpixelorder = ZM_SUBPIX_ORDER_RGBA; pixelFormat = AV_PIX_FMT_RGBA; } else if ( zm_is_rgb24(pixelFormat) ) { subpixelorder = ZM_SUBPIX_ORDER_RGB; pixelFormat = AV_PIX_FMT_RGB24; } else if ( pixelFormat == AV_PIX_FMT_GRAY8 ) { subpixelorder = ZM_SUBPIX_ORDER_NONE; pixelFormat = AV_PIX_FMT_GRAY8; } else { Panic("Unexpected pixel format %d (%s); legacy colours=%d subpixelorder=%d", pixelFormat, zm_get_pix_fmt_name(pixelFormat), colours, subpixelorder); } packet = av_packet_ptr{av_packet_alloc()}; } // FfmpegCamera::FfmpegCamera FfmpegCamera::~FfmpegCamera() { Close(); FFMPEGDeInit(); } int FfmpegCamera::PrimeCapture() { start_read_time = std::chrono::steady_clock::now(); Close(); mVideoStreamId = -1; mAudioStreamId = -1; Debug(1, "Priming capture from %s", mMaskedPath.c_str()); return OpenFfmpeg(); } int FfmpegCamera::PreCapture() { return 0; } bool FfmpegCamera::loopSeekToStart(AVFormatContext *ctx) { if (!(ctx->pb && ctx->pb->seekable)) { Debug(1, "loop: input is not seekable, cannot loop"); return false; } // Bump the per-stream absolute offsets so the next packet continues one frame // after the last emitted dts. mLastVideoDTS/mLastAudioDTS already include the // offset that was in effect, and start_time is the raw container start, so // this accumulates correctly across repeated loops. // The audio stream lives in the secondary context only when one is in use; // otherwise both streams share mFormatContext. bool ctxHasVideo = (mVideoStream != nullptr) && (ctx == mFormatContext); bool ctxHasAudio = (mAudioStream != nullptr) && (ctx == (mSecondFormatContext ? mSecondFormatContext : mFormatContext)); if (ctxHasVideo && (mLastVideoDTS != AV_NOPTS_VALUE)) { int64_t start = (mVideoStream->start_time != AV_NOPTS_VALUE) ? mVideoStream->start_time : 0; mLoopVideoOffset = mLastVideoDTS + mLoopVideoFrameDuration - start; } if (ctxHasAudio && (mLastAudioDTS != AV_NOPTS_VALUE)) { int64_t start = (mAudioStream->start_time != AV_NOPTS_VALUE) ? mAudioStream->start_time : 0; mLoopAudioOffset = mLastAudioDTS + mLoopAudioFrameDuration - start; } int ret = avformat_seek_file(ctx, -1, INT64_MIN, 0, INT64_MAX, AVSEEK_FLAG_BACKWARD); if (ret < 0) { ret = av_seek_frame(ctx, -1, 0, AVSEEK_FLAG_BACKWARD); } if (ret < 0) { Warning("loop: seek to start failed: %s", av_make_error_string(ret).c_str()); return false; } Debug(1, "loop: sought to start; offsets video=%" PRId64 " audio=%" PRId64, mLoopVideoOffset, mLoopAudioOffset); return true; } int FfmpegCamera::readFrameWithLoop(AVFormatContext *ctx, AVPacket *pkt) { int ret = av_read_frame(ctx, pkt); if (ret >= 0) return ret; bool eof = (ret == AVERROR_EOF) || (ctx->pb && ctx->pb->eof_reached); if (eof && mLoop && loopSeekToStart(ctx)) { Info("loop: reached end of input, restarting from beginning"); ret = av_read_frame(ctx, pkt); } return ret; } RealtimePaceDecision ComputeRealtimePace( int64_t ts_us, int64_t anchor_ts_us, Microseconds elapsed, Microseconds cap) { // Backward movement (e.g. a discontinuity the jump checks let through): // re-anchor so we never try to sleep on a negative delta. if (ts_us < anchor_ts_us) return {true, Microseconds(0)}; Microseconds target_elapsed(ts_us - anchor_ts_us); Microseconds delay = target_elapsed - elapsed; // Already behind schedule: deliver immediately. if (delay <= Microseconds(0)) return {false, Microseconds(0)}; // A large gap usually means a timestamp discontinuity rather than a genuine // multi-second frame interval; re-anchor instead of stalling the capture. if (delay > cap) return {true, Microseconds(0)}; return {false, delay}; } void FfmpegCamera::paceRealtime(int64_t ts_us) { if (!mRealtime || (ts_us == AV_NOPTS_VALUE)) return; TimePoint now = std::chrono::steady_clock::now(); // Anchor on the first packet. if (!mRealtimeAnchored) { mRealtimeStartWall = now; mRealtimeStartTS = ts_us; mRealtimeAnchored = true; return; } Microseconds elapsed = std::chrono::duration_cast(now - mRealtimeStartWall); RealtimePaceDecision d = ComputeRealtimePace(ts_us, mRealtimeStartTS, elapsed, Seconds(10)); if (d.reanchor) { Debug(1, "realtime: re-anchoring pacing at ts %" PRId64 "us", ts_us); mRealtimeStartWall = now; mRealtimeStartTS = ts_us; return; } if (d.sleep > Microseconds(0)) { Debug(4, "realtime: sleeping %" PRId64 "us to pace to stream rate", static_cast(d.sleep.count())); std::this_thread::sleep_for(d.sleep); } } int FfmpegCamera::Capture(std::shared_ptr &zm_packet) { if (!mIsPrimed) return -1; start_read_time = std::chrono::steady_clock::now(); int ret; AVFormatContext *formatContextPtr; int64_t lastPTS = -1; if ( mSecondFormatContext and mAudioStream and ( av_rescale_q(mLastAudioPTS, mAudioStream->time_base, AV_TIME_BASE_Q) < av_rescale_q(mLastVideoPTS, mVideoStream->time_base, AV_TIME_BASE_Q) ) ) { // if audio stream is behind video stream, then read from audio, otherwise video formatContextPtr = mSecondFormatContext; lastPTS = mLastAudioPTS; Debug(4, "Using audio input because audio PTS %" PRId64 " < video PTS %" PRId64, av_rescale_q(mLastAudioPTS, mAudioStream->time_base, AV_TIME_BASE_Q), av_rescale_q(mLastVideoPTS, mVideoStream->time_base, AV_TIME_BASE_Q) ); if ((ret = readFrameWithLoop(formatContextPtr, packet.get())) < 0) { if ( // Check if EOF. (ret == AVERROR_EOF || (formatContextPtr->pb && formatContextPtr->pb->eof_reached)) || // Check for Connection failure. (ret == -110) ) { Info("Unable to read packet from stream %d: error %d \"%s\".", packet->stream_index, ret, av_make_error_string(ret).c_str()); } else { logPrintf(Logger::ERROR + monitor->Importance(), "Unable to read packet from stream %d: error %d \"%s\".", packet->stream_index, ret, av_make_error_string(ret).c_str()); } return -1; } } else { formatContextPtr = mFormatContext; Debug(4, "Using video input because %" PRId64 " >= %" PRId64, (mAudioStream?av_rescale_q(mLastAudioPTS, mAudioStream->time_base, AV_TIME_BASE_Q):0), av_rescale_q(mLastVideoPTS, mVideoStream->time_base, AV_TIME_BASE_Q) ); if ((ret = readFrameWithLoop(formatContextPtr, packet.get())) < 0) { if ( // Check if EOF. (ret == AVERROR_EOF || (formatContextPtr->pb && formatContextPtr->pb->eof_reached)) || // Check for Connection failure. (ret == -110) ) { Info("Unable to read packet from stream %d: error %d \"%s\".", packet->stream_index, ret, av_make_error_string(ret).c_str()); } else { logPrintf(Logger::ERROR + monitor->Importance(), "Unable to read packet from stream %d: error %d \"%s\".", packet->stream_index, ret, av_make_error_string(ret).c_str()); } return -1; } if ( packet->stream_index == mVideoStreamId) { lastPTS = mLastVideoPTS; } else if (packet->stream_index == mAudioStreamId) { lastPTS = mLastAudioPTS; } else { Debug(1, "Have packet (%d) which isn't for video (%d) or audio stream (%d).", packet->stream_index, mVideoStreamId, mAudioStreamId); return 0; } } AVStream *stream = formatContextPtr->streams[packet->stream_index]; // Loop mode: shift this packet's timestamps by the accumulated per-stream // offset so they stay monotonically increasing across loop restarts. Done // before the pts/dts backward-jump checks and before the packet is queued, so // every consumer (analysis, recording) sees continuous timestamps. Also keep // the per-stream frame duration up to date for spacing the next loop. if (mLoop) { int64_t off = 0; if (packet->stream_index == mVideoStreamId) { off = mLoopVideoOffset; if (packet->duration > 0) mLoopVideoFrameDuration = packet->duration; } else if (packet->stream_index == mAudioStreamId) { off = mLoopAudioOffset; if (packet->duration > 0) mLoopAudioFrameDuration = packet->duration; } if (off) { if (packet->pts != AV_NOPTS_VALUE) packet->pts += off; if (packet->dts != AV_NOPTS_VALUE) packet->dts += off; } } ZM_DUMP_STREAM_PACKET(stream, packet, "ffmpeg_camera in"); if ((packet->pts != AV_NOPTS_VALUE) and (lastPTS >= 0)) { if (packet->pts < 0) { // 32-bit wrap around? Info("Suspected 32bit wraparound in input pts. %" PRId64, packet->pts); return -1; } else if (packet->pts - lastPTS < -10*stream->time_base.den) { if (!monitor->WallClockTimestamps()) { // -10 is for 10 seconds. Avigilon cameras seem to jump around by about 36 constantly double pts_time = static_cast(av_rescale_q(packet->pts, stream->time_base, AV_TIME_BASE_Q)) / AV_TIME_BASE; double last_pts_time = static_cast(av_rescale_q(lastPTS, stream->time_base, AV_TIME_BASE_Q)) / AV_TIME_BASE; logPrintf(Logger::WARNING + monitor->Importance(), "Stream pts jumped back in time too far. pts %.2f - last pts %.2f = %.2f > 10seconds", pts_time, last_pts_time, pts_time - last_pts_time); } if (error_count > 5) return -1; error_count += 1; return 0; } } // Check DTS for significant backward jumps. Some cameras/encoders produce // non-monotonic DTS (B-frames, stream restarts) that PTS checks won't catch. if (packet->dts != AV_NOPTS_VALUE) { int64_t lastDTS = (packet->stream_index == mVideoStreamId) ? mLastVideoDTS : mLastAudioDTS; if (lastDTS != AV_NOPTS_VALUE) { int64_t dts_delta = packet->dts - lastDTS; if (dts_delta < -10*stream->time_base.den) { double dts_time = static_cast(av_rescale_q(packet->dts, stream->time_base, AV_TIME_BASE_Q)) / AV_TIME_BASE; double last_dts_time = static_cast(av_rescale_q(lastDTS, stream->time_base, AV_TIME_BASE_Q)) / AV_TIME_BASE; logPrintf(Logger::WARNING + monitor->Importance(), "Stream dts jumped back in time too far. dts %.2f - last dts %.2f = %.2f > 10seconds stream %d", dts_time, last_dts_time, dts_time - last_dts_time, packet->stream_index); if (error_count > 5) return -1; error_count += 1; return 0; } } } av_packet_guard pkt_guard{packet}; // Real-time pacing: throttle delivery to the stream's native rate. Use dts // (monotonic in read order) when available, otherwise pts. Sleep happens here, // after all the drop/error filters above, so only packets we actually deliver // advance the schedule. if (mRealtime) { int64_t pace_ts = (packet->dts != AV_NOPTS_VALUE) ? packet->dts : packet->pts; if (pace_ts != AV_NOPTS_VALUE) paceRealtime(av_rescale_q(pace_ts, stream->time_base, AV_TIME_BASE_Q)); } zm_packet->codec_type = stream->codecpar->codec_type; bytes += packet->size; zm_packet->set_packet(packet.get()); zm_packet->stream = stream; zm_packet->pts = av_rescale_q(packet->pts, stream->time_base, AV_TIME_BASE_Q); if (packet->pts != AV_NOPTS_VALUE) { if (stream == mVideoStream) { if (mFirstVideoPTS == AV_NOPTS_VALUE) mFirstVideoPTS = packet->pts; mLastVideoPTS = packet->pts - mFirstVideoPTS; } else if (stream == mAudioStream) { if (mFirstAudioPTS == AV_NOPTS_VALUE) mFirstAudioPTS = packet->pts; mLastAudioPTS = packet->pts - mFirstAudioPTS; } } if (packet->dts != AV_NOPTS_VALUE) { if (packet->stream_index == mVideoStreamId) mLastVideoDTS = packet->dts; else if (packet->stream_index == mAudioStreamId) mLastAudioDTS = packet->dts; } return 1; } // FfmpegCamera::Capture int FfmpegCamera::PostCapture() { // Nothing to do here return 0; } int FfmpegCamera::OpenFfmpeg() { int ret = 0; error_count = 0; #if LIBAVFORMAT_VERSION_CHECK(59, 16, 100, 16, 100) const #endif AVInputFormat *input_format = nullptr; // Handle options AVDictionary *opts = nullptr; if (!mOptions.empty()) { ret = av_dict_parse_string(&opts, mOptions.c_str(), "=", kOptionSeparators, 0); if (ret < 0) { Warning("Could not parse ffmpeg input options '%s'", mOptions.c_str()); } // "loop=1" is handled by us (seek-to-start on EOF), not by ffmpeg, which // does not understand it for most demuxers. Consume it so it is not passed // through and reported as an unrecognized option below. AVDictionaryEntry *loop_entry = av_dict_get(opts, "loop", nullptr, 0); if (loop_entry) { mLoop = (loop_entry->value != nullptr) && (atoi(loop_entry->value) != 0); av_dict_set(&opts, "loop", nullptr, 0); Debug(1, "Loop-on-EOF mode %s from options", mLoop ? "enabled" : "disabled"); } // "realtime=1" (alias "re=1") is handled by us, like ffmpeg's -re flag: // pace packet delivery to the stream's native rate rather than reading the // file as fast as possible. Consume it so it is not passed through to the // demuxer and reported as an unrecognized option below. AVDictionaryEntry *re_entry = av_dict_get(opts, "realtime", nullptr, 0); if (!re_entry) re_entry = av_dict_get(opts, "re", nullptr, 0); if (re_entry) { mRealtime = (re_entry->value != nullptr) && (atoi(re_entry->value) != 0); av_dict_set(&opts, "realtime", nullptr, 0); av_dict_set(&opts, "re", nullptr, 0); Debug(1, "Real-time pacing %s from options", mRealtime ? "enabled" : "disabled"); } } // Fresh prime: drop any real-time anchor so pacing restarts cleanly, and // start with no loop offset. mRealtimeAnchored = false; mLoopVideoOffset = 0; mLoopAudioOffset = 0; // Set transport method as specified by method field, rtpUni is default std::string protocol = mPath.substr(0, 4); protocol = StringToUpper(protocol); if ( protocol == "RTSP" ) { const std::string method = Method(); if ( method == "rtpMulti" ) { ret = av_dict_set(&opts, "rtsp_transport", "udp_multicast", 0); } else if ( method == "rtpRtsp" ) { ret = av_dict_set(&opts, "rtsp_transport", "tcp", 0); } else if ( method == "rtpRtspHttp" ) { ret = av_dict_set(&opts, "rtsp_transport", "http", 0); } else if ( method == "rtpUni" ) { ret = av_dict_set(&opts, "rtsp_transport", "udp", 0); } else { Warning("Unknown method (%s)", method.c_str()); } if (ret < 0) { Warning("Could not set rtsp_transport method '%s'", method.c_str()); } } else if (protocol == "V4L2") { avdevice_register_all(); input_format = av_find_input_format("video4linux2"); if (!input_format) { Error("Cannot find v4l2 input format"); return -1; } mPath = mPath.substr(7); } // end if RTSP Debug(1, "Calling avformat_open_input for %s", mMaskedPath.c_str()); mFormatContext = avformat_alloc_context(); if (!mFormatContext) { Error("Unable to allocate format context"); av_dict_free(&opts); return -1; } mFormatContext->interrupt_callback.callback = FfmpegInterruptCallback; mFormatContext->interrupt_callback.opaque = this; mFormatContext->flags |= AVFMT_FLAG_NOBUFFER | AVFMT_FLAG_FLUSH_PACKETS; if (mUser.length() > 0) { try { Url url(mPath); if (url.user_info().empty()) { url.user_info(mUser + ":" + mPass); mPath = url.str(); Debug(1, "Rebuilt URI with encoded parameters: '%s'", mMaskedPath.c_str()); } } catch (const Url::parse_error &e) { Debug(1, "Could not parse path as URL: %s", e.what()); } } ret = avformat_open_input(&mFormatContext, mPath.c_str(), input_format, &opts); if (ret != 0) { logPrintf(Logger::ERROR + monitor->Importance(), "Unable to open input %s due to: %s", mMaskedPath.c_str(), av_make_error_string(ret).c_str()); avformat_close_input(&mFormatContext); mFormatContext = nullptr; av_dict_free(&opts); return -1; } AVDictionaryEntry *e = nullptr; while ((e = av_dict_get(opts, "", e, AV_DICT_IGNORE_SUFFIX)) != nullptr) { Warning("Option %s not recognized by ffmpeg", e->key); } av_dict_free(&opts); ret = avformat_find_stream_info(mFormatContext, nullptr); if (ret < 0) { Error("Unable to find stream info from %s due to: %s", mMaskedPath.c_str(), av_make_error_string(ret).c_str()); avformat_close_input(&mFormatContext); return -1; } // Find first video stream present, the one we want Might not be the first mVideoStreamId = -1; mAudioStreamId = -1; for (unsigned int i=0; i < mFormatContext->nb_streams; i++) { AVStream *stream = mFormatContext->streams[i]; zm_dump_stream_format(mFormatContext, i, 0, 0); if (is_video_stream(stream)) { if (!(stream->codecpar->width && stream->codecpar->height)) { Warning("No width and height in video stream. Trying again"); continue; } if (mVideoStreamId == -1) { mVideoStreamId = i; mVideoStream = stream; } else { Debug(2, "Have another video stream."); if (stream->codecpar->width == width and stream->codecpar->height == height) { Debug(1, "Choosing alternate video stream because it matches our resolution."); mVideoStreamId = i; mVideoStream = stream; } else { stream->discard = AVDISCARD_ALL; } } } else if (is_audio_stream(stream)) { if (mAudioStreamId == -1) { mAudioStreamId = i; mAudioStream = mFormatContext->streams[i]; } else { Debug(2, "Have another audio stream."); } } else { Debug(1, "Unknown stream type for stream %d", i); } } // end foreach stream if (mVideoStreamId == -1) { avformat_close_input(&mFormatContext); return -1; } Debug(3, "Found video stream at index %d, audio stream at index %d", mVideoStreamId, mAudioStreamId); const AVCodec *mVideoCodec = nullptr; std::listcodec_data = get_decoder_data(mVideoStream->codecpar->codec_id, monitor->DecoderName().c_str()); if (codec_data.size() == 0 and (!monitor->DecoderName().empty() and (monitor->DecoderName() != "auto"))) { Warning("No decoder for codec %d found with name %s. Trying auto.", mVideoStream->codecpar->codec_id, monitor->DecoderName().c_str()); codec_data = get_decoder_data(mVideoStream->codecpar->codec_id, "auto"); } for (auto it = codec_data.begin(); it != codec_data.end(); it ++) { const CodecData *chosen_codec_data = *it; Debug(1, "Found codec %s", chosen_codec_data->codec_name); mVideoCodec = avcodec_find_decoder_by_name(chosen_codec_data->codec_name); if (!mVideoCodec) { mVideoCodec = avcodec_find_decoder(mVideoStream->codecpar->codec_id); if (!mVideoCodec) { // Try and get the codec from the codec context Error("Can't find codec for video stream from %s", mMaskedPath.c_str()); continue; } } mVideoCodecContext = avcodec_alloc_context3(mVideoCodec); avcodec_parameters_to_context(mVideoCodecContext, mFormatContext->streams[mVideoStreamId]->codecpar); mVideoCodecContext->framerate = mVideoStream->r_frame_rate; mVideoCodecContext->sw_pix_fmt = chosen_codec_data->sw_pix_fmt; // libavcodec defaults thread_count to 1, making software 1080p H.264 // decode hit ~60ms/frame and saturate a core per camera. Default to 2 // frame-threads instead. User can override (including 0 = auto) via // thread_count in monitor Options. mVideoCodecContext->thread_count = 2; mVideoCodecContext->thread_type = FF_THREAD_FRAME | FF_THREAD_SLICE; // Set default options for this codec if (chosen_codec_data->options_defaults) { AVDictionary *opts_defaults = nullptr; av_dict_parse_string(&opts_defaults, chosen_codec_data->options_defaults, "=", ",", 0); AVDictionaryEntry *e = nullptr; while ((e = av_dict_get(opts_defaults, "", e, AV_DICT_IGNORE_SUFFIX)) != nullptr) { const AVDictionaryEntry *entry = av_dict_get(opts, e->key, nullptr, AV_DICT_MATCH_CASE); if (!entry) { int ret; if ((ret = av_dict_set(&opts, e->key, e->value, 0)) < 0) { Error("Couldn't set default Option %s set to %s", e->key, e->value); } Debug(1, "Option %s set to %s from default", e->key, e->value); } } av_dict_free(&opts_defaults); } //Set user-specified options, which may override codec defaults if (!mOptions.empty()) { av_dict_parse_string(&opts, mOptions.c_str(), "=", kOptionSeparators, 0); const AVDictionaryEntry *entry = av_dict_get(opts, "thread_count", nullptr, AV_DICT_MATCH_CASE); if (entry) { mVideoCodecContext->thread_count = std::stoul(entry->value); Debug(1, "Setting codec thread_count to %d", mVideoCodecContext->thread_count); av_dict_set(&opts, "thread_count", nullptr, AV_DICT_MATCH_CASE); } // reorder_queparse for avforpts, mOpcodec av_dict_set(&opts, "reorder_queue_size", nullptr, AV_DICT_MATCH_CASE); av_dict_set(&opts, "probesize", nullptr, AV_DICT_MATCH_CASE); // loop / realtime (re) are consumed by FfmpegCamera, not the decoder; // strip them so avcodec_open2 doesn't report them as unrecognized. av_dict_set(&opts, "loop", nullptr, AV_DICT_MATCH_CASE); av_dict_set(&opts, "realtime", nullptr, AV_DICT_MATCH_CASE); av_dict_set(&opts, "re", nullptr, AV_DICT_MATCH_CASE); } if (use_hwaccel && (hwaccel_name != "")) { #if HAVE_LIBAVUTIL_HWCONTEXT_H // 3.2 doesn't seem to have all the bits in place, so let's require 3.4 and up #if LIBAVCODEC_VERSION_CHECK(57, 107, 0, 107, 0) // Build the list of hw device types to try. A DecoderHWAccelName of // "auto" probes every hwaccel libav offers and uses the first that both // the decoder supports and whose device can be created; any other value // is a comma-separated priority list of device-type names, tried in // order (e.g. "cuda,vaapi"; a single name like "vaapi" is just the // one-element case and behaves as before). If nothing usable is found // we transparently fall back to software. std::vector candidate_types; bool auto_detect = (hwaccel_name == "auto"); enum AVHWDeviceType it = AV_HWDEVICE_TYPE_NONE; while ((it = av_hwdevice_iterate_types(it)) != AV_HWDEVICE_TYPE_NONE) { Debug(1, "Available hwdevice type %s", av_hwdevice_get_type_name(it)); if (auto_detect) candidate_types.push_back(it); } if (!auto_detect) { for (const std::string &token : Split(hwaccel_name, ',')) { std::string name = TrimSpaces(token); if (name.empty()) continue; enum AVHWDeviceType named = av_hwdevice_find_type_by_name(name.c_str()); if (named == AV_HWDEVICE_TYPE_NONE) Warning("Unknown hwaccel device type '%s', skipping.", name.c_str()); else candidate_types.push_back(named); } } for (enum AVHWDeviceType type : candidate_types) { Debug(1, "Trying hwdevice %s", av_hwdevice_get_type_name(type)); hw_pix_fmt = AV_PIX_FMT_NONE; #if LIBAVUTIL_VERSION_CHECK(56, 22, 0, 14, 0) // Does this decoder advertise a hw config for this device type? for (int i = 0;; i++) { const AVCodecHWConfig *config = avcodec_get_hw_config(mVideoCodec, i); if (!config) break; if ((config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX) && (config->device_type == type)) { hw_pix_fmt = config->pix_fmt; Debug(1, "Decoder %s supports type %s (pix_fmt %s).", mVideoCodec->name, av_hwdevice_get_type_name(type), zm_get_pix_fmt_name(hw_pix_fmt)); } } // end foreach hwconfig #else hw_pix_fmt = find_fmt_by_hw_type(type); #endif if (hw_pix_fmt == AV_PIX_FMT_NONE) { Debug(1, "Decoder %s has no hw_pix_fmt for %s, skipping.", mVideoCodec->name, av_hwdevice_get_type_name(type)); continue; } ret = av_hwdevice_ctx_create(&hw_device_ctx, type, (hwaccel_device != "" ? hwaccel_device.c_str() : nullptr), nullptr, 0); if (ret < 0 and hwaccel_device != "") ret = av_hwdevice_ctx_create(&hw_device_ctx, type, nullptr, nullptr, 0); if (ret < 0) { Warning("Failed to create %s hwaccel device: %s", av_hwdevice_get_type_name(type), av_make_error_string(ret).c_str()); hw_pix_fmt = AV_PIX_FMT_NONE; hw_device_ctx = nullptr; continue; // try the next candidate } // Success: wire up hardware decoding and stop searching. Info("Using %s hardware decoding for %s", av_hwdevice_get_type_name(type), mVideoCodec->name); mVideoCodecContext->hwaccel_flags |= AV_HWACCEL_FLAG_IGNORE_LEVEL; //if (!lavc_param->check_hw_profile) mVideoCodecContext->hwaccel_flags |= AV_HWACCEL_FLAG_ALLOW_PROFILE_MISMATCH; // Set opaque to point to our hw_pix_fmt so callback can access it mVideoCodecContext->opaque = &hw_pix_fmt; mVideoCodecContext->get_format = get_hw_format; mVideoCodecContext->hw_device_ctx = av_buffer_ref(hw_device_ctx); break; } // end foreach candidate type if (hw_pix_fmt == AV_PIX_FMT_NONE) { // "auto" asks us to probe, so finding nothing is unremarkable. But if // the admin named hwaccels explicitly and none of them worked, this // monitor now decodes on the CPU and nothing else will ever say so: // a default install does not log Debug, and we only get here once, at // capture start. That silence hides a dead GPU for as long as nobody // wonders why the load average doubled. if (auto_detect) { Debug(1, "No usable hardware decoder found; falling back to software decoding."); } else { Warning("None of the requested hwaccels (%s) are usable for %s; " "falling back to software decoding.", hwaccel_name.c_str(), mVideoCodec->name); } use_hwaccel = false; } #else Debug(1, "AVCodec not new enough for hwaccel"); #endif #else Warning("HWAccel support not compiled in."); #endif } // end if hwaccel_name ret = avcodec_open2(mVideoCodecContext, mVideoCodec, &opts); e = nullptr; while ((e = av_dict_get(opts, "", e, AV_DICT_IGNORE_SUFFIX)) != nullptr) { Warning("Option %s not recognized by ffmpeg", e->key); } av_dict_free(&opts); if (ret < 0) { Error("Unable to open codec for video stream from %s", mMaskedPath.c_str()); avcodec_free_context(&mVideoCodecContext); mVideoCodecContext = nullptr; continue; } Debug(1, "Thread count? %d", mVideoCodecContext->thread_count); zm_dump_codec(mVideoCodecContext); break; } // end foreach codec if (!mVideoCodecContext) { Debug(1, "Failed with known codecs, trying harder"); mVideoCodecContext = open_fallback_decoder(mVideoStream->codecpar, &mVideoCodec); } if (!mVideoCodecContext) { Warning("Failed to open codec"); return -1; } if (mAudioStreamId >= 0) { const AVCodec *mAudioCodec = nullptr; if (!(mAudioCodec = avcodec_find_decoder(mAudioStream->codecpar->codec_id))) { Debug(1, "Can't find codec for audio stream from %s", mMaskedPath.c_str()); } else { mAudioCodecContext = avcodec_alloc_context3(mAudioCodec); avcodec_parameters_to_context(mAudioCodecContext, mAudioStream->codecpar); zm_dump_stream_format((mSecondFormatContext?mSecondFormatContext:mFormatContext), mAudioStreamId, 0, 0); // Open the codec if (avcodec_open2(mAudioCodecContext, mAudioCodec, nullptr) < 0) { Error("Unable to open codec for audio stream from %s", mMaskedPath.c_str()); return -1; } // end if opened } // end if found decoder } else if (!monitor->GetSecondPath().empty()) { Debug(1, "Trying secondary stream at %s", mMaskedSecondPath.c_str()); std::string secondPath = mSecondPath; if (mUser.length() > 0) { try { Url url(mSecondPath); if (url.user_info().empty()) { url.user_info(mUser + ":" + mPass); secondPath = url.str(); Debug(1, "Rebuilt secondary URI with encoded parameters"); } else { Debug(1, "Secondary path already has authentication, not overriding"); } } catch (const Url::parse_error &e) { Debug(1, "Could not parse secondary path as URL: %s", e.what()); } } mSecondInput = zm::make_unique(); if (mSecondInput->Open(secondPath.c_str()) > 0) { mSecondFormatContext = mSecondInput->get_format_context(); mAudioStreamId = mSecondInput->get_audio_stream_id(); mAudioStream = mSecondInput->get_audio_stream(); mAudioCodecContext = mSecondInput->get_audio_codec_context(); } else { Warning("Failed to open secondary input"); } } // end if have audio stream if ( ((unsigned int)mVideoCodecContext->width != width) || ((unsigned int)mVideoCodecContext->height != height) ) { Debug(1, "Monitor dimensions are %dx%d but camera is sending %dx%d", width, height, mVideoCodecContext->width, mVideoCodecContext->height); } // Seed fallback per-frame durations (in each stream's time_base) used to space // loops apart when a packet's own duration is missing. Live values from // packet->duration override these as packets are read. if (mVideoStream && mVideoStream->avg_frame_rate.num > 0) { mLoopVideoFrameDuration = av_rescale_q(1, av_inv_q(mVideoStream->avg_frame_rate), mVideoStream->time_base); } if (mLoopVideoFrameDuration <= 0) mLoopVideoFrameDuration = 1; if (mAudioStream && mAudioStream->codecpar->sample_rate > 0) { // Typical AAC frame is 1024 samples; good enough as a fallback spacing. mLoopAudioFrameDuration = av_rescale_q(1024, av_make_q(1, mAudioStream->codecpar->sample_rate), mAudioStream->time_base); } if (mLoopAudioFrameDuration <= 0) mLoopAudioFrameDuration = 1; mIsPrimed = true; return 1; } // int FfmpegCamera::OpenFfmpeg() int FfmpegCamera::Close() { mIsPrimed = false; mLastVideoPTS = 0; mLastAudioPTS = 0; mLastVideoDTS = AV_NOPTS_VALUE; mLastAudioDTS = AV_NOPTS_VALUE; if (mVideoCodecContext) { //avcodec_close(mVideoCodecContext); avcodec_free_context(&mVideoCodecContext); mVideoCodecContext = nullptr; } if (mAudioCodecContext and !mSecondInput) { // If second input, then these will get freed in FFmpeg_Input's destructor //avcodec_close(mAudioCodecContext); avcodec_free_context(&mAudioCodecContext); mAudioCodecContext = nullptr; } #if HAVE_LIBAVUTIL_HWCONTEXT_H if ( hw_device_ctx ) { av_buffer_unref(&hw_device_ctx); } #endif // Re-arm hardware decoding for the next PrimeCapture(). Without this a // single transient failure - a GPU still resetting, a driver that finished // initialising after zmc started, a render node not yet permissioned - // pins the monitor to software decoding for the whole life of the process, // long after the hardware is healthy again. Reconnects are the natural // retry point, so let them retry. use_hwaccel = true; if ( mFormatContext ) { avformat_close_input(&mFormatContext); mFormatContext = nullptr; } return 0; } // end FfmpegCamera::Close int FfmpegCamera::FfmpegInterruptCallback(void *ctx) { if (zm_terminate) { Debug(1, "Received terminate in cb"); return zm_terminate; } TimePoint now = std::chrono::steady_clock::now(); if (now - start_read_time > Seconds(10)) { Debug(1, "timeout in ffmpeg camera now %" PRIi64 " - %" PRIi64 " > 10 s", static_cast(std::chrono::duration_cast(now.time_since_epoch()).count()), static_cast(std::chrono::duration_cast(start_read_time.time_since_epoch()).count())); return 1; } return 0; }