From c8d274edac70837cdeff5d32e4ed1ec1882d65c9 Mon Sep 17 00:00:00 2001
From: derrod <dennis@obsproject.com>
Date: Fri, 31 Mar 2023 15:50:17 +0200
Subject: [PATCH] UI: Only use FFmpeg compat check for external codecs

For all the codecs we ship with OBS we can use a hardcoded lookup
instead as the FFmpeg one is unreliable.
---
 UI/window-basic-settings.cpp | 79 ++++++++++++++++++------------------
 1 file changed, 39 insertions(+), 40 deletions(-)

diff --git a/UI/window-basic-settings.cpp b/UI/window-basic-settings.cpp
index 8fccaf986..5c10d22c0 100644
--- a/UI/window-basic-settings.cpp
+++ b/UI/window-basic-settings.cpp
@@ -23,6 +23,7 @@
 #include <initializer_list>
 #include <sstream>
 #include <unordered_map>
+#include <unordered_set>
 #include <QCompleter>
 #include <QGuiApplication>
 #include <QLineEdit>
@@ -4933,6 +4934,37 @@ void OBSBasicSettings::AdvOutSplitFileChanged()
 	ui->advOutSplitFileSize->setVisible(splitFileType == 1);
 }
 
+static const unordered_set<string> builtin_codecs = {
+	"h264", "hevc", "av1",       "prores",    "aac",
+	"opus", "alac", "pcm_s16le", "pcm_s24le", "pcm_f32le"};
+
+static const unordered_map<string, unordered_set<string>> codec_compat = {
+	// Technically our muxer supports HEVC and AV1 as well, but nothing else does
+	{"flv", {"h264", "aac"}},
+	{"ts", {"h264", "hevc", "aac", "opus"}},
+	{"m3u8",
+	 {"h264", "hevc", "aac"}}, // Also using MPEG-TS, but no Opus support
+	{"mov",
+	 {"h264", "hevc", "prores", "aac", "alac", "pcm_s16le", "pcm_s24le",
+	  "pcm_f32le"}},
+	{"mp4", {"h264", "hevc", "av1", "aac", "opus", "alac", "flac"}},
+	// MKV supports everything
+	{"mkv", {}},
+};
+
+static bool ContainerSupportsCodec(const string &container, const string &codec)
+{
+	auto iter = codec_compat.find(container);
+	if (iter == codec_compat.end())
+		return false;
+
+	auto codecs = iter->second;
+	// Assume everything is supported
+	if (codecs.empty())
+		return true;
+	return codecs.count(codec) > 0;
+}
+
 static void DisableIncompatibleCodecs(QComboBox *cbox, const string &format,
 				      const QString &streamEncoder)
 {
@@ -4954,20 +4986,11 @@ static void DisableIncompatibleCodecs(QComboBox *cbox, const string &format,
 							   encoderId.c_str());
 		const char *codec = obs_get_encoder_codec(encoderId.c_str());
 
-		bool is_compatible = false;
-		/* FFmpeg's check does not work for MPEG-TS and MKV. */
-		if (format == "ts") {
-			is_compatible = strcmp(codec, "aac") == 0 ||
-					strcmp(codec, "opus") == 0 ||
-					strcmp(codec, "hevc") == 0 ||
-					strcmp(codec, "h264") == 0;
-		} else if (format == "mkv") {
-			/* MKV eats everything. */
-			is_compatible = true;
-		} else {
+		bool is_compatible = ContainerSupportsCodec(format, codec);
+		/* Fall back to FFmpeg check if codec not one of the built-in ones. */
+		if (!is_compatible && !builtin_codecs.count(codec))
 			is_compatible = ff_format_codec_compatible(
 				codec, format.c_str());
-		}
 
 		QStandardItemModel *model =
 			dynamic_cast<QStandardItemModel *>(cbox->model());
@@ -5011,8 +5034,6 @@ void OBSBasicSettings::AdvOutRecCheckCodecs()
 	/* Remove leading "f" for fragmented MP4/MOV */
 	if (format == "fmp4" || format == "fmov")
 		format = format.erase(0, 1);
-	else if (format == "m3u8")
-		format = "hls";
 
 	QString streamEncoder = ui->advOutEncoder->currentData().toString();
 
@@ -5655,23 +5676,12 @@ static void DisableIncompatibleSimpleCodecs(QComboBox *cbox,
 			codec = obs_get_encoder_codec(encoder_id);
 		}
 
-		bool is_compatible = true;
-		if (format == "flv") {
-			/* If FLV, only H.264 and AAC are compatible */
-			is_compatible = codec == "aac" || codec == "h264";
-		} else if (format == "mov" || format == "fmov") {
-			/* If MOV, Opus is not compatible */
-			is_compatible = codec != "opus";
-		} else if (format == "ts") {
-			/* If MPEG-TS, AV1 is incompatible */
-			is_compatible = codec != "av1";
-		}
-
 		QStandardItemModel *model =
 			dynamic_cast<QStandardItemModel *>(cbox->model());
 		QStandardItem *item = model->item(idx);
 
-		if (is_compatible) {
+		if (ContainerSupportsCodec(QT_TO_UTF8(format),
+					   QT_TO_UTF8(codec))) {
 			item->setFlags(Qt::ItemIsSelectable |
 				       Qt::ItemIsEnabled);
 		} else {
@@ -5701,23 +5711,12 @@ static void DisableIncompatibleSimpleContainer(QComboBox *cbox,
 	for (int idx = 0; idx < cbox->count(); idx++) {
 		QString format = cbox->itemData(idx).toString();
 
-		bool is_compatible = true;
-		if (format == "flv") {
-			/* If flv, ónly H.264 and AAC are compatible */
-			is_compatible = aCodec == "aac" && vCodec == "h264";
-		} else if (format == "mov" || format == "fmov") {
-			/* If MOV, Opus is not compatible */
-			is_compatible = aCodec != "opus";
-		} else if (format == "ts") {
-			/* If MPEG-TS, AV1 is incompatible */
-			is_compatible = vCodec != "av1";
-		}
-
 		QStandardItemModel *model =
 			dynamic_cast<QStandardItemModel *>(cbox->model());
 		QStandardItem *item = model->item(idx);
 
-		if (is_compatible) {
+		if (ContainerSupportsCodec(QT_TO_UTF8(format),
+					   QT_TO_UTF8(vCodec))) {
 			item->setFlags(Qt::ItemIsSelectable |
 				       Qt::ItemIsEnabled);
 		} else {