From f52d8bddcc48be09c4a5d64677dcf4d15c92a282 Mon Sep 17 00:00:00 2001
From: Richard Stanway <r1ch@r1ch.net>
Date: Sat, 30 Mar 2019 15:45:23 +0100
Subject: [PATCH] libobs: Allow Win32 pipes to pass STDERR for logging of
 errors

---
 libobs/util/pipe-posix.c   |  9 +++++++++
 libobs/util/pipe-windows.c | 41 +++++++++++++++++++++++++++++++++++---
 libobs/util/pipe.h         |  2 ++
 3 files changed, 49 insertions(+), 3 deletions(-)

diff --git a/libobs/util/pipe-posix.c b/libobs/util/pipe-posix.c
index 195f6e0cd..3d776864c 100644
--- a/libobs/util/pipe-posix.c
+++ b/libobs/util/pipe-posix.c
@@ -73,6 +73,15 @@ size_t os_process_pipe_read(os_process_pipe_t *pp, uint8_t *data, size_t len)
 	return fread(data, 1, len, pp->file);
 }
 
+size_t os_process_pipe_read_err(os_process_pipe_t *pp, uint8_t *data, size_t len)
+{
+	/* XXX: unsupported on posix */
+	UNUSED_PARAMETER(pp);
+	UNUSED_PARAMETER(data);
+	UNUSED_PARAMETER(len);
+	return 0;
+}
+
 size_t os_process_pipe_write(os_process_pipe_t *pp, const uint8_t *data,
 		size_t len)
 {
diff --git a/libobs/util/pipe-windows.c b/libobs/util/pipe-windows.c
index d3d35e225..5510072af 100644
--- a/libobs/util/pipe-windows.c
+++ b/libobs/util/pipe-windows.c
@@ -24,6 +24,7 @@
 struct os_process_pipe {
 	bool read_pipe;
 	HANDLE handle;
+	HANDLE handle_err;
 	HANDLE process;
 };
 
@@ -42,7 +43,7 @@ static bool create_pipe(HANDLE *input, HANDLE *output)
 }
 
 static inline bool create_process(const char *cmd_line, HANDLE stdin_handle,
-		HANDLE stdout_handle, HANDLE *process)
+		HANDLE stdout_handle, HANDLE stderr_handle, HANDLE *process)
 {
 	PROCESS_INFORMATION pi = {0};
 	wchar_t *cmd_line_w = NULL;
@@ -53,6 +54,7 @@ static inline bool create_process(const char *cmd_line, HANDLE stdin_handle,
 	si.dwFlags = STARTF_USESTDHANDLES | STARTF_FORCEOFFFEEDBACK;
 	si.hStdInput = stdin_handle;
 	si.hStdOutput = stdout_handle;
+	si.hStdError = stderr_handle;
 
 	os_utf8_to_wcs_ptr(cmd_line, 0, &cmd_line_w);
 	if (cmd_line_w) {
@@ -77,6 +79,7 @@ os_process_pipe_t *os_process_pipe_create(const char *cmd_line,
 	bool read_pipe;
 	HANDLE process;
 	HANDLE output;
+	HANDLE err_input, err_output;
 	HANDLE input;
 	bool success;
 
@@ -90,26 +93,38 @@ os_process_pipe_t *os_process_pipe_create(const char *cmd_line,
 		return NULL;
 	}
 
+	if (!create_pipe(&err_input, &err_output)) {
+		return NULL;
+	}
+
 	read_pipe = *type == 'r';
 
 	success = !!SetHandleInformation(read_pipe ? input : output,
-			HANDLE_FLAG_INHERIT, false);
+		HANDLE_FLAG_INHERIT, false);
+	if (!success) {
+		goto error;
+	}
+
+	success = !!SetHandleInformation(err_input, HANDLE_FLAG_INHERIT, false);
 	if (!success) {
 		goto error;
 	}
 
 	success = create_process(cmd_line, read_pipe ? NULL : input,
-			read_pipe ? output : NULL, &process);
+			read_pipe ? output : NULL, err_output, &process);
 	if (!success) {
 		goto error;
 	}
 
 	pp = bmalloc(sizeof(*pp));
+
 	pp->handle = read_pipe ? input : output;
 	pp->read_pipe = read_pipe;
 	pp->process = process;
+	pp->handle_err = err_input;
 
 	CloseHandle(read_pipe ? output : input);
+	CloseHandle(err_output);
 	return pp;
 
 error:
@@ -126,6 +141,7 @@ int os_process_pipe_destroy(os_process_pipe_t *pp)
 		DWORD code;
 
 		CloseHandle(pp->handle);
+		CloseHandle(pp->handle_err);
 
 		WaitForSingleObject(pp->process, INFINITE);
 		if (GetExitCodeProcess(pp->process, &code))
@@ -158,6 +174,25 @@ size_t os_process_pipe_read(os_process_pipe_t *pp, uint8_t *data, size_t len)
 	return 0;
 }
 
+size_t os_process_pipe_read_err(os_process_pipe_t *pp, uint8_t *data, size_t len)
+{
+	DWORD bytes_read;
+	bool success;
+
+	if (!pp || !pp->handle_err) {
+		return 0;
+	}
+
+	success = !!ReadFile(pp->handle_err, data, (DWORD)len, &bytes_read, NULL);
+	if (success && bytes_read) {
+		return bytes_read;
+	}
+	else
+		bytes_read = GetLastError();
+
+	return 0;
+}
+
 size_t os_process_pipe_write(os_process_pipe_t *pp, const uint8_t *data,
 		size_t len)
 {
diff --git a/libobs/util/pipe.h b/libobs/util/pipe.h
index cb595fdb1..5c4451475 100644
--- a/libobs/util/pipe.h
+++ b/libobs/util/pipe.h
@@ -27,5 +27,7 @@ EXPORT int os_process_pipe_destroy(os_process_pipe_t *pp);
 
 EXPORT size_t os_process_pipe_read(os_process_pipe_t *pp, uint8_t *data,
 		size_t len);
+EXPORT size_t os_process_pipe_read_err(os_process_pipe_t *pp, uint8_t *data,
+	size_t len);
 EXPORT size_t os_process_pipe_write(os_process_pipe_t *pp, const uint8_t *data,
 		size_t len);