// // ZoneMinder Ffmpeg Camera Class Implementation // Copyright (C) 2001-2008 Philip Coombes // // This program is free software; you can redistribute it and/or // modify it under the terms of the GNU General Public License // as published by the Free Software Foundation; either version 2 // of the License, or (at your option) any later version. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU General Public License // along with this program; if not, write to the Free Software // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. // #include "zm_ffmpeg_camera.h" #include "zm_ffmpeg_input.h" #include "zm_monitor.h" #include "zm_packet.h" #include "zm_signal.h" #include "zm_utils.h" extern "C" { #include } TimePoint start_read_time; #if HAVE_LIBAVUTIL_HWCONTEXT_H #if LIBAVCODEC_VERSION_CHECK(57, 89, 0, 89, 0) static enum AVPixelFormat hw_pix_fmt; static enum AVPixelFormat get_hw_format( AVCodecContext *ctx, const enum AVPixelFormat *pix_fmts ) { const enum AVPixelFormat *p; for ( p = pix_fmts; *p != -1; p++ ) { if ( *p == hw_pix_fmt ) return *p; } Error("Failed to get HW surface format for %s.", av_get_pix_fmt_name(hw_pix_fmt)); for ( p = pix_fmts; *p != -1; p++ ) Error("Available HW surface format was %s.", av_get_pix_fmt_name(*p)); return AV_PIX_FMT_NONE; } #if !LIBAVUTIL_VERSION_CHECK(56, 22, 0, 14, 0) static enum AVPixelFormat find_fmt_by_hw_type(const enum AVHWDeviceType type) { switch (type) { case AV_HWDEVICE_TYPE_VAAPI: return AV_PIX_FMT_VAAPI; case AV_HWDEVICE_TYPE_DXVA2: return AV_PIX_FMT_DXVA2_VLD; case AV_HWDEVICE_TYPE_D3D11VA: return AV_PIX_FMT_D3D11; case AV_HWDEVICE_TYPE_VDPAU: return AV_PIX_FMT_VDPAU; case AV_HWDEVICE_TYPE_CUDA: return AV_PIX_FMT_CUDA; #ifdef AV_HWDEVICE_TYPE_MMAL case AV_HWDEVICE_TYPE_MMAL: return AV_PIX_FMT_MMAL; #endif case AV_HWDEVICE_TYPE_VIDEOTOOLBOX: return AV_PIX_FMT_VIDEOTOOLBOX; default: return AV_PIX_FMT_NONE; } } #endif #endif #endif FfmpegCamera::FfmpegCamera( const Monitor *monitor, const std::string &p_path, const std::string &p_second_path, const std::string &p_method, const std::string &p_options, int p_width, int p_height, int p_colours, int p_brightness, int p_contrast, int p_hue, int p_colour, bool p_capture, bool p_record_audio, const std::string &p_hwaccel_name, const std::string &p_hwaccel_device) : Camera( monitor, FFMPEG_SRC, p_width, p_height, p_colours, ZM_SUBPIX_ORDER_DEFAULT_FOR_COLOUR(p_colours), p_brightness, p_contrast, p_hue, p_colour, p_capture, p_record_audio ), mPath(p_path), mSecondPath(p_second_path), mMethod(p_method), mOptions(p_options), hwaccel_name(p_hwaccel_name), hwaccel_device(p_hwaccel_device) { if ( capture ) { FFMPEGInit(); } frameCount = 0; mCanCapture = false; error_count = 0; use_hwaccel = true; #if HAVE_LIBAVUTIL_HWCONTEXT_H hw_device_ctx = nullptr; #if LIBAVCODEC_VERSION_CHECK(57, 89, 0, 89, 0) hw_pix_fmt = AV_PIX_FMT_NONE; #endif #endif mConvertContext = nullptr; /* Has to be located inside the constructor so other components such as zma * will receive correct colours and subpixel order */ if ( colours == ZM_COLOUR_RGB32 ) { subpixelorder = ZM_SUBPIX_ORDER_RGBA; imagePixFormat = AV_PIX_FMT_RGBA; } else if ( colours == ZM_COLOUR_RGB24 ) { subpixelorder = ZM_SUBPIX_ORDER_RGB; imagePixFormat = AV_PIX_FMT_RGB24; } else if ( colours == ZM_COLOUR_GRAY8 ) { subpixelorder = ZM_SUBPIX_ORDER_NONE; imagePixFormat = AV_PIX_FMT_GRAY8; } else { Panic("Unexpected colours: %d", colours); } } // FfmpegCamera::FfmpegCamera FfmpegCamera::~FfmpegCamera() { Close(); FFMPEGDeInit(); } int FfmpegCamera::PrimeCapture() { start_read_time = std::chrono::steady_clock::now(); if ( mCanCapture ) { Debug(1, "Priming capture from %s, Closing", mPath.c_str()); Close(); } mVideoStreamId = -1; mAudioStreamId = -1; Debug(1, "Priming capture from %s", mPath.c_str()); return OpenFfmpeg(); } int FfmpegCamera::PreCapture() { return 0; } int FfmpegCamera::Capture(std::shared_ptr &zm_packet) { if (!mCanCapture) return -1; start_read_time = std::chrono::steady_clock::now(); int ret; AVFormatContext *formatContextPtr; if ( mSecondFormatContext and ( av_rescale_q(mLastAudioPTS, mAudioStream->time_base, AV_TIME_BASE_Q) < av_rescale_q(mLastVideoPTS, mVideoStream->time_base, AV_TIME_BASE_Q) ) ) { // if audio stream is behind video stream, then read from audio, otherwise video formatContextPtr = mSecondFormatContext; Debug(4, "Using audio input because audio PTS %" PRId64 " < video PTS %" PRId64, av_rescale_q(mLastAudioPTS, mAudioStream->time_base, AV_TIME_BASE_Q), av_rescale_q(mLastVideoPTS, mVideoStream->time_base, AV_TIME_BASE_Q) ); } else { formatContextPtr = mFormatContext; Debug(4, "Using video input because %" PRId64 " >= %" PRId64, (mAudioStream?av_rescale_q(mLastAudioPTS, mAudioStream->time_base, AV_TIME_BASE_Q):0), av_rescale_q(mLastVideoPTS, mVideoStream->time_base, AV_TIME_BASE_Q) ); } if ((ret = av_read_frame(formatContextPtr, &packet)) < 0) { if ( // Check if EOF. (ret == AVERROR_EOF || (formatContextPtr->pb && formatContextPtr->pb->eof_reached)) || // Check for Connection failure. (ret == -110) ) { Info("Unable to read packet from stream %d: error %d \"%s\".", packet.stream_index, ret, av_make_error_string(ret).c_str()); } else { Error("Unable to read packet from stream %d: error %d \"%s\".", packet.stream_index, ret, av_make_error_string(ret).c_str()); } return -1; } AVStream *stream = formatContextPtr->streams[packet.stream_index]; ZM_DUMP_STREAM_PACKET(stream, packet, "ffmpeg_camera in"); zm_packet->codec_type = stream->codecpar->codec_type; bytes += packet.size; zm_packet->set_packet(&packet); zm_packet->stream = stream; zm_packet->pts = av_rescale_q(packet.pts, stream->time_base, AV_TIME_BASE_Q); if (packet.pts != AV_NOPTS_VALUE) { if (stream == mVideoStream) { if (mFirstVideoPTS == AV_NOPTS_VALUE) mFirstVideoPTS = packet.pts; mLastVideoPTS = packet.pts - mFirstVideoPTS; } else { if (mFirstAudioPTS == AV_NOPTS_VALUE) mFirstAudioPTS = packet.pts; mLastAudioPTS = packet.pts - mFirstAudioPTS; } } zm_av_packet_unref(&packet); return 1; } // FfmpegCamera::Capture int FfmpegCamera::PostCapture() { // Nothing to do here return 0; } int FfmpegCamera::OpenFfmpeg() { int ret; error_count = 0; // Handle options AVDictionary *opts = nullptr; ret = av_dict_parse_string(&opts, Options().c_str(), "=", ",", 0); if ( ret < 0 ) { Warning("Could not parse ffmpeg input options '%s'", Options().c_str()); } // Set transport method as specified by method field, rtpUni is default std::string protocol = mPath.substr(0, 4); protocol = StringToUpper(protocol); if ( protocol == "RTSP" ) { const std::string method = Method(); if ( method == "rtpMulti" ) { ret = av_dict_set(&opts, "rtsp_transport", "udp_multicast", 0); } else if ( method == "rtpRtsp" ) { ret = av_dict_set(&opts, "rtsp_transport", "tcp", 0); } else if ( method == "rtpRtspHttp" ) { ret = av_dict_set(&opts, "rtsp_transport", "http", 0); } else if ( method == "rtpUni" ) { ret = av_dict_set(&opts, "rtsp_transport", "udp", 0); } else { Warning("Unknown method (%s)", method.c_str()); } if ( ret < 0 ) { Warning("Could not set rtsp_transport method '%s'", method.c_str()); } } // end if RTSP Debug(1, "Calling avformat_open_input for %s", mPath.c_str()); mFormatContext = avformat_alloc_context(); // Speed up find_stream_info // FIXME can speed up initial analysis but need sensible parameters... // mFormatContext->probesize = 32; // mFormatContext->max_analyze_duration = 32; mFormatContext->interrupt_callback.callback = FfmpegInterruptCallback; mFormatContext->interrupt_callback.opaque = this; ret = avformat_open_input(&mFormatContext, mPath.c_str(), nullptr, &opts); if ( ret != 0 ) { logPrintf(Logger::ERROR + monitor->Importance(), "Unable to open input %s due to: %s", mPath.c_str(), av_make_error_string(ret).c_str()); if ( mFormatContext ) { avformat_close_input(&mFormatContext); mFormatContext = nullptr; } av_dict_free(&opts); return -1; } AVDictionaryEntry *e = nullptr; while ( (e = av_dict_get(opts, "", e, AV_DICT_IGNORE_SUFFIX)) != nullptr ) { Warning("Option %s not recognized by ffmpeg", e->key); } av_dict_free(&opts); Debug(1, "Finding stream info"); ret = avformat_find_stream_info(mFormatContext, nullptr); if ( ret < 0 ) { Error("Unable to find stream info from %s due to: %s", mPath.c_str(), av_make_error_string(ret).c_str()); return -1; } // Find first video stream present // The one we want Might not be the first mVideoStreamId = -1; mAudioStreamId = -1; for (unsigned int i=0; i < mFormatContext->nb_streams; i++) { AVStream *stream = mFormatContext->streams[i]; if (is_video_stream(stream)) { if (mVideoStreamId == -1) { mVideoStreamId = i; mVideoStream = mFormatContext->streams[i]; // if we break, then we won't find the audio stream continue; } else { Debug(2, "Have another video stream."); } } else if (is_audio_stream(stream)) { if (mAudioStreamId == -1) { mAudioStreamId = i; mAudioStream = mFormatContext->streams[i]; } else { Debug(2, "Have another audio stream."); } } } // end foreach stream if (mVideoStreamId == -1) { Error("Unable to locate video stream in %s", mPath.c_str()); return -1; } Debug(3, "Found video stream at index %d, audio stream at index %d", mVideoStreamId, mAudioStreamId); AVCodec *mVideoCodec = nullptr; if (mVideoStream->codecpar->codec_id == AV_CODEC_ID_H264) { if ((mVideoCodec = avcodec_find_decoder_by_name("h264_mmal")) == nullptr) { Debug(1, "Failed to find decoder (h264_mmal)"); } else { Debug(1, "Success finding decoder (h264_mmal)"); } } if (!mVideoCodec) { mVideoCodec = avcodec_find_decoder(mVideoStream->codecpar->codec_id); if (!mVideoCodec) { // Try and get the codec from the codec context Error("Can't find codec for video stream from %s", mPath.c_str()); return -1; } } mVideoCodecContext = avcodec_alloc_context3(mVideoCodec); avcodec_parameters_to_context(mVideoCodecContext, mFormatContext->streams[mVideoStreamId]->codecpar); #ifdef CODEC_FLAG2_FAST mVideoCodecContext->flags2 |= CODEC_FLAG2_FAST | CODEC_FLAG_LOW_DELAY; #endif zm_dump_stream_format(mFormatContext, mVideoStreamId, 0, 0); if (use_hwaccel && (hwaccel_name != "")) { #if HAVE_LIBAVUTIL_HWCONTEXT_H // 3.2 doesn't seem to have all the bits in place, so let's require 3.4 and up #if LIBAVCODEC_VERSION_CHECK(57, 107, 0, 107, 0) // Print out available types enum AVHWDeviceType type = AV_HWDEVICE_TYPE_NONE; while ((type = av_hwdevice_iterate_types(type)) != AV_HWDEVICE_TYPE_NONE) Debug(1, "%s", av_hwdevice_get_type_name(type)); const char *hw_name = hwaccel_name.c_str(); type = av_hwdevice_find_type_by_name(hw_name); if (type == AV_HWDEVICE_TYPE_NONE) { Debug(1, "Device type %s is not supported.", hw_name); } else { Debug(1, "Found hwdevice %s", av_hwdevice_get_type_name(type)); } #if LIBAVUTIL_VERSION_CHECK(56, 22, 0, 14, 0) // Get hw_pix_fmt for (int i = 0;; i++) { const AVCodecHWConfig *config = avcodec_get_hw_config(mVideoCodec, i); if (!config) { Debug(1, "Decoder %s does not support config %d.", mVideoCodec->name, i); break; } if ((config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX) && (config->device_type == type) ) { hw_pix_fmt = config->pix_fmt; Debug(1, "Decoder %s does support our type %s.", mVideoCodec->name, av_hwdevice_get_type_name(type)); //break; } else { Debug(1, "Decoder %s hwConfig doesn't match our type: %s != %s, pix_fmt %s.", mVideoCodec->name, av_hwdevice_get_type_name(type), av_hwdevice_get_type_name(config->device_type), av_get_pix_fmt_name(config->pix_fmt) ); } } // end foreach hwconfig #else hw_pix_fmt = find_fmt_by_hw_type(type); #endif if (hw_pix_fmt != AV_PIX_FMT_NONE) { Debug(1, "Selected hw_pix_fmt %d %s", hw_pix_fmt, av_get_pix_fmt_name(hw_pix_fmt)); mVideoCodecContext->hwaccel_flags |= AV_HWACCEL_FLAG_IGNORE_LEVEL; //if (!lavc_param->check_hw_profile) mVideoCodecContext->hwaccel_flags |= AV_HWACCEL_FLAG_ALLOW_PROFILE_MISMATCH; ret = av_hwdevice_ctx_create(&hw_device_ctx, type, (hwaccel_device != "" ? hwaccel_device.c_str() : nullptr), nullptr, 0); if ( ret < 0 and hwaccel_device != "" ) { ret = av_hwdevice_ctx_create(&hw_device_ctx, type, nullptr, nullptr, 0); } if (ret < 0) { Error("Failed to create hwaccel device. %s", av_make_error_string(ret).c_str()); hw_pix_fmt = AV_PIX_FMT_NONE; } else { Debug(1, "Created hwdevice for %s", hwaccel_device.c_str()); mVideoCodecContext->get_format = get_hw_format; mVideoCodecContext->hw_device_ctx = av_buffer_ref(hw_device_ctx); } } else { Debug(1, "Failed to find suitable hw_pix_fmt."); } #else Debug(1, "AVCodec not new enough for hwaccel"); #endif #else Warning("HWAccel support not compiled in."); #endif } // end if hwaccel_name // set codec to automatically determine how many threads suits best for the decoding job mVideoCodecContext->thread_count = 0; if (mVideoCodec->capabilities | AV_CODEC_CAP_FRAME_THREADS) { mVideoCodecContext->thread_type = FF_THREAD_FRAME; } else if (mVideoCodec->capabilities | AV_CODEC_CAP_SLICE_THREADS) { mVideoCodecContext->thread_type = FF_THREAD_SLICE; } else { mVideoCodecContext->thread_count = 1; //don't use multithreading } ret = avcodec_open2(mVideoCodecContext, mVideoCodec, &opts); e = nullptr; while ((e = av_dict_get(opts, "", e, AV_DICT_IGNORE_SUFFIX)) != nullptr) { Warning("Option %s not recognized by ffmpeg", e->key); } if (ret < 0) { Error("Unable to open codec for video stream from %s", mPath.c_str()); av_dict_free(&opts); return -1; } zm_dump_codec(mVideoCodecContext); if (mAudioStreamId == -1 and !monitor->GetSecondPath().empty()) { Debug(1, "Trying secondary stream at %s", monitor->GetSecondPath().c_str()); FFmpeg_Input *second_input = new FFmpeg_Input(); if (second_input->Open(monitor->GetSecondPath().c_str()) > 0) { mSecondFormatContext = second_input->get_format_context(); mAudioStreamId = second_input->get_audio_stream_id(); mAudioStream = second_input->get_audio_stream(); } else { Warning("Failed to open secondary input"); } } // end if have audio stream if ( mAudioStreamId >= 0 ) { AVCodec *mAudioCodec = nullptr; if (!(mAudioCodec = avcodec_find_decoder(mAudioStream->codecpar->codec_id))) { Debug(1, "Can't find codec for audio stream from %s", mPath.c_str()); } else { mAudioCodecContext = avcodec_alloc_context3(mAudioCodec); avcodec_parameters_to_context(mAudioCodecContext, mAudioStream->codecpar); zm_dump_stream_format((mSecondFormatContext?mSecondFormatContext:mFormatContext), mAudioStreamId, 0, 0); // Open the codec if (avcodec_open2(mAudioCodecContext, mAudioCodec, nullptr) < 0) { Error("Unable to open codec for audio stream from %s", mPath.c_str()); return -1; } // end if opened } // end if found decoder } // end if mAudioStreamId if ( ((unsigned int)mVideoCodecContext->width != width) || ((unsigned int)mVideoCodecContext->height != height) ) { Warning("Monitor dimensions are %dx%d but camera is sending %dx%d", width, height, mVideoCodecContext->width, mVideoCodecContext->height); } mCanCapture = true; return 1; } // int FfmpegCamera::OpenFfmpeg() int FfmpegCamera::Close() { mCanCapture = false; if ( mVideoCodecContext ) { avcodec_close(mVideoCodecContext); avcodec_free_context(&mVideoCodecContext); mVideoCodecContext = nullptr; // Freed by av_close_input_file } if ( mAudioCodecContext ) { avcodec_close(mAudioCodecContext); avcodec_free_context(&mAudioCodecContext); mAudioCodecContext = nullptr; // Freed by av_close_input_file } #if HAVE_LIBAVUTIL_HWCONTEXT_H if ( hw_device_ctx ) { av_buffer_unref(&hw_device_ctx); } #endif if ( mFormatContext ) { avformat_close_input(&mFormatContext); mFormatContext = nullptr; } return 0; } // end FfmpegCamera::Close int FfmpegCamera::FfmpegInterruptCallback(void *ctx) { if (zm_terminate) { Debug(1, "Received terminate in cb"); return zm_terminate; } TimePoint now = std::chrono::steady_clock::now(); if (now - start_read_time > Seconds(10)) { Debug(1, "timeout in ffmpeg camera now %" PRIi64 " - %" PRIi64 " > 10 s", static_cast(std::chrono::duration_cast(now.time_since_epoch()).count()), static_cast(std::chrono::duration_cast(start_read_time.time_since_epoch()).count())); return 1; } return 0; }