From 3715154a8ef7e5af10a3b3317e2418133af22e25 Mon Sep 17 00:00:00 2001
From: Robin Gareus <robin@gareus.org>
Date: Sat, 25 Mar 2023 06:09:43 +0100
Subject: [PATCH] Allow concurrent child processes

When multiple child processes are running, closing the
stdin of one child did not send EOF or cause POLLNVAL,
as long as a dup()'ed filedes still existed.

This fixes an issue when running an mp3 encoder while the
video monitor is visible, and will allow to concurrently
run multiple mp3 encoders or other child processes.

Previously this caused Ardour to hang indefinitely in CmdPipeWriter
```
 _proc->close_stdin();
 _proc->wait(); // << here
```
---
 libs/pbd/system_exec.cc   |  1 +
 libs/vfork/exec_wrapper.c | 37 +++++++++++++++++++++++++++++++++++--
 2 files changed, 36 insertions(+), 2 deletions(-)

diff --git a/libs/pbd/system_exec.cc b/libs/pbd/system_exec.cc
index 48f9ea1908..96be624b34 100644
--- a/libs/pbd/system_exec.cc
+++ b/libs/pbd/system_exec.cc
@@ -920,6 +920,7 @@ SystemExec::close_stdin()
 	if (pin[1] < 0) {
 		return;
 	}
+	fsync (pin[1]);
 	close_fd (pin[0]);
 	close_fd (pin[1]);
 	fsync (pout[0]);
diff --git a/libs/vfork/exec_wrapper.c b/libs/vfork/exec_wrapper.c
index 75d3187de1..0453262b5a 100644
--- a/libs/vfork/exec_wrapper.c
+++ b/libs/vfork/exec_wrapper.c
@@ -21,6 +21,9 @@
 #include <string.h>
 #include <fcntl.h>
 #include <signal.h>
+#include <errno.h>
+#include <sys/time.h>
+#include <sys/resource.h>
 
 #ifndef STDIN_FILENO
 #define STDIN_FILENO 0
@@ -35,6 +38,32 @@
 extern char **environ;
 static void close_fd (int *fd) { if ((*fd) >= 0) close (*fd); *fd = -1; }
 
+static int close_allv (const int except_fds[]) {
+	struct rlimit rl;
+
+	if (getrlimit (RLIMIT_NOFILE, &rl) < 0) {
+		return -1;
+	}
+
+	for (int fd = 0; fd < (int) rl.rlim_max; fd++) {
+		if (fd <= 3) {
+			continue;
+		}
+
+		for (int i = 0; except_fds[i] >= 0; i++) {
+			if (except_fds[i] == fd) {
+				continue;
+			}
+		}
+
+		if (close (fd) < 0 && errno != EBADF) {
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
 int main(int argc, char *argv[]) {
 	if (argc < 10) {
 		// TODO: if argv > 3, assume pok[] is given, notifify parent.
@@ -53,6 +82,8 @@ int main(int argc, char *argv[]) {
 	pout[0] = atoi(argv[5]);
 	pout[1] = atoi(argv[6]);
 
+	int good_fds[3] = { pok[1], pout[1], -1 };
+
 	int stderr_mode = atoi(argv[7]);
 	int nicelevel = atoi(argv[8]);
 
@@ -106,8 +137,10 @@ int main(int argc, char *argv[]) {
 	signal(SIGPIPE, SIG_DFL);
 #endif
 
-	/* all systems go */
-	execve(argv[9], &argv[9], envp);
+	if (0 == close_allv (good_fds)) {
+		/* all systems go */
+		execve(argv[9], &argv[9], envp);
+	}
 
 	/* if we reach here something went wrong.. */
 	char buf = 0;