resolve gtk + VST threading issues

while (gtk_events_pending()) gtk_main_iteration(); never returns
as long as there's a idle call registered somewhere (and it's not
called from an idle callback itself)

gtk2_ardour/ardour_ui.cc

@@ -81,6 +81,10 @@
 #include "ardour/session_utils.h"
 #include "ardour/slave.h"
 
+#ifdef WINDOWS_VST_SUPPORT
+#include <fst.h>
+#endif
+
 #include "timecode/time.h"
 
 typedef uint64_t microseconds_t;
@@ -1008,6 +1012,9 @@ If you still wish to quit, please use the\n\n\
 
 	halt_connection.disconnect ();
 	AudioEngine::instance()->stop ();
+#ifdef WINDOWS_VST_SUPPORT
+	fst_stop_threading();
+#endif
 	quit ();
 }
 
@@ -2916,7 +2923,13 @@ ARDOUR_UI::load_session (const std::string& path, const std::string& snap_name,
 		_session->set_clean ();
 	}
 
+#ifdef WINDOWS_VST_SUPPORT
+	fst_stop_threading();
+#endif
 	flush_pending ();
+#ifdef WINDOWS_VST_SUPPORT
+	fst_start_threading();
+#endif
 	retval = 0;
 
   out:
@@ -3343,10 +3356,6 @@ ARDOUR_UI::add_route (Gtk::Window* float_window)
 
 	PBD::ScopedConnection idle_connection;
 
-	if (count > 8) {
-		ARDOUR::GUIIdle.connect (idle_connection, MISSING_INVALIDATOR, boost::bind (&Gtkmm2ext::UI::flush_pending, this), gui_context());
-	}
-
 	string template_path = add_route_dialog->track_template();
 
 	if (!template_path.empty()) {

gtk2_ardour/editor_audio_import.cc

@@ -632,7 +632,7 @@ Editor::embed_sndfiles (vector<string> paths, bool multifile,
 				goto out;
 			}
 
-			ARDOUR_UI::instance()->flush_pending ();
+			gtk_main_iteration();
 		}
 	}
 

libs/ardour/session_vst.cc

@@ -28,6 +28,9 @@
 #include "ardour/windows_vst_plugin.h"
 #include "ardour/vestige/aeffectx.h"
 #include "ardour/vst_types.h"
+#ifdef WINDOWS_VST_SUPPORT
+#include <fst.h>
+#endif
 
 #include "i18n.h"
 
@@ -100,26 +103,9 @@ intptr_t Session::vst_callback (
 
 	case audioMasterIdle:
 		SHOW_CALLBACK ("amc: audioMasterIdle\n");
-		// call application idle routine (this will
-		// call effEditIdle for all open editors too)
-
-#if 0 // TODO -> emit to GUI OR better delegete to fst/fst
-
-		// This allows the main GUI window to update if needed.
-		// Some plugins take over the GUI event loop
-		// which causes the main GUI to freeze while the plugin GUI continues to run. This code
-		// prevents the main GUI from being frozen.
-
-		do {
-#ifdef GDK_WINDOWING_X11
-			gtk_main_iteration_do(false);
-#else
-			gtk_main_iteration()
+#ifdef WINDOWS_VST_SUPPORT
+		fst_audio_master_idle();
 #endif
-		} while (gtk_events_pending());
-#endif
-		printf("audioMasterIdle\n");
-
 		if (effect) {
 			effect->dispatcher(effect, effEditIdle, 0, 0, NULL, 0.0f);
 		}

libs/fst/fst.c

@@ -15,7 +15,7 @@ fst_error (const char *fmt, ...)
 	va_end (ap);
 }
 
-void
+static void
 default_fst_error_callback (const char *desc)
 {
 	fprintf(stderr, "%s\n", desc);

libs/fst/fst.h

@@ -48,6 +48,9 @@ extern void fst_move_window_into_view (VSTState *);
 extern VSTInfo *fst_get_info (char *dllpathname);
 extern void fst_free_info (VSTInfo *info);
 extern void fst_event_loop_remove_plugin (VSTState* fst);
+extern void fst_start_threading(void);
+extern void fst_stop_threading(void);
+extern void fst_audio_master_idle(void);
 
 #ifdef __cplusplus
 }

libs/fst/vstwin.c

@@ -16,6 +16,7 @@ static UINT_PTR idle_timer_id   = 0;
 #include <wine/exception.h>
 #include <pthread.h>
 static int gui_quit = 0;
+static unsigned int idle_id = 0;
 
 #endif
 
@@ -249,12 +250,14 @@ fst_handle_new (void)
 #ifndef PLATFORM_WINDOWS /* linux + wine */
 static gboolean
 g_idle_call (gpointer ignored) {
+	if (gui_quit) return FALSE;
 	MSG msg;
 	if (PeekMessageA (&msg, NULL, 0, 0, 1)) {
 		TranslateMessage (&msg);
 		DispatchMessageA (&msg);
 	}
 	idle_hands(NULL, 0, 0, 0);
+	g_main_context_iteration(NULL, FALSE);
 	return gui_quit ? FALSE : TRUE;
 }
 #endif
@@ -311,14 +314,33 @@ fst_init (void* possible_hmodule)
 		fst_error ("Error in fst_init(): (class registration failed");
 		return -1;
 	}
-#ifndef PLATFORM_WINDOWS /* linux + wine */
-	gui_quit = 0;
-	g_idle_add (g_idle_call, NULL); // XXX too early ?
-	//g_timeout_add(40, g_idle_call, NULL);
-#endif
 	return 0;
 }
 
+void
+fst_start_threading(void)
+{
+#ifndef PLATFORM_WINDOWS /* linux + wine */
+	if (idle_id == 0) {
+		gui_quit = 0;
+		idle_id = g_idle_add (g_idle_call, NULL);
+	}
+#endif
+}
+
+void
+fst_stop_threading(void) {
+#ifndef PLATFORM_WINDOWS /* linux + wine */
+	if (idle_id != 0) {
+		gui_quit = 1;
+		PostQuitMessage (0);
+		g_main_context_iteration(NULL, FALSE);
+		//g_source_remove(idle_id);
+		idle_id = 0;
+	}
+#endif
+}
+
 void
 fst_exit (void)
 {
@@ -333,8 +355,10 @@ fst_exit (void)
 		KillTimer(NULL, idle_timer_id);
 	}
 #else /* linux + wine */
-	gui_quit = 1;
-	PostQuitMessage (0);
+	if (idle_id) {
+		gui_quit = 1;
+		PostQuitMessage (0);
+	}
 #endif
 
 	host_initialized = FALSE;
@@ -562,6 +586,10 @@ fst_instantiate (VSTHandle* fhandle, audioMasterCallback amc, void* userptr)
 	return fst;
 }
 
+void fst_audio_master_idle(void) {
+	while(g_main_context_iteration(NULL, FALSE)) ;
+}
+
 void
 fst_close (VSTState* fst)
 {