Make execstack optional, and check clang's variant with space

see also ec5b06e63d
and https://discourse.ardour.org/t/fyi-ardour-9-doesnt-build-on-linux-using-clang/111896

gtk2_ardour/wscript

@@ -655,7 +655,7 @@ def build(bld):
         'LOCALEDIR="' + os.path.normpath(bld.env['LOCALEDIR']) + '"',
         ]
     obj.install_path = bld.env['DLLDIR']
-    obj.linkflags    = ''
+    obj.linkflags    = bld.env['compiler_flags_dict']['execstack']
     obj.uselib       = 'UUID FLAC FONTCONFIG GTHREAD OGG PANGOMM CURL DL CANVAS FFTW3F LO TAGLIB XML LILV RUBBERBAND AUBIO LRDF ARCHIVE VAMPSDK VAMPHOSTSDK'
 
     if bld.is_defined('YTK'):
@@ -679,7 +679,6 @@ def build(bld):
         obj.source += [ 'bundle_env_mingw.cc' ]
         obj.source += [ 'windows_icon.rc' ]
     else:
-        obj.linkflags += ' -zexecstack'
         obj.source += [ 'bundle_env_linux.cc' ]
         obj.use += [ 'X11' ]
 

headless/wscript

@@ -21,7 +21,7 @@ def build(bld):
     obj.source    = hardour_sources
     obj.target = 'hardour-' + bld.env['VERSION']
     obj.includes = ['.']
-    obj.linkflags= ''
+    obj.linkflags= bld.env['compiler_flags_dict']['execstack']
     obj.use      = [ 'libpbd',
                      'libardour',
                      'libardour_cp',
@@ -55,8 +55,6 @@ def build(bld):
     elif bld.env['build_target'] == 'mingw':
         if bld.env['DEBUG'] == False:
             obj.linkflags += ' -mwindows'
-    else:
-        obj.linkflags += ' -zexecstack'
 
     obj.includes += ['../libs']
 

libs/fst/wscript

@@ -19,7 +19,7 @@ def build(bld):
         obj.source    = 'vst3-scanner.cc'
         obj.target    = 'ardour-vst3-scanner'
         obj.includes  = [ '../pbd/', '../ardour/', '../vst3/', '..' ]
-        obj.linkflags = ''
+        obj.linkflags = bld.env['compiler_flags_dict']['execstack']
         obj.defines   = [
             'VST3_SCANNER_APP',
             'VERSIONSTRING="' + bld.env['VERSION'] + '"',
@@ -38,8 +38,6 @@ def build(bld):
         elif bld.env['build_target'] == 'mingw':
             obj.uselib += ' GDI32'
             obj.linkflags += ' -mwindows'
-        else:
-            obj.linkflags += ' -zexecstack'
 
         if re.search ("bsd", sys.platform) != None:
             obj.defines.append('_POSIX_C_SOURCE=200809L')

libs/vfork/wscript

@@ -16,12 +16,8 @@ def build(bld):
     obj.source = 'exec_wrapper.c'
     obj.target = 'ardour-exec-wrapper'
     obj.install_path = os.path.join(bld.env['LIBDIR'])
+    obj.linkflags    = bld.env['compiler_flags_dict']['execstack']
     obj.defines = [
             '_POSIX_SOURCE',
             '_XOPEN_SOURCE=500',
     ]
-
-    if sys.platform == 'darwin':
-        pass
-    else:
-        obj.linkflags = ' -zexecstack'

luasession/wscript

@@ -34,7 +34,7 @@ def build(bld):
     obj.source       = 'luasession.cc'
     obj.target       = 'luasession'
     obj.includes     = ['../libs']
-    obj.linkflags    = ''
+    obj.linkflags    = bld.env['compiler_flags_dict']['execstack']
     obj.use          = ['liblua',
                         'libpbd',
                         'libardour',
@@ -69,8 +69,6 @@ def build(bld):
         obj.use    += ' libappleutility'
     elif bld.env['build_target'] == 'mingw':
         obj.linkflags += ' -mwindows'
-    else:
-        obj.linkflags += ' -zexecstack'
 
     if bld.is_defined('NEED_INTL'):
         obj.linkflags += ' -lintl'

session_utils/wscript

@@ -25,7 +25,7 @@ def build_ardour_util(bld, util):
     obj.source   = ['common.cc', util + '.cc' ]
     obj.target   = pgmprefix + '-' + util
     obj.includes = ['.']
-    obj.linkflags= ''
+    obj.linkflags= bld.env['compiler_flags_dict']['execstack']
     obj.use      = [ 'libpbd',
                      'libardour',
                      'libardour_cp',
@@ -59,8 +59,6 @@ def build_ardour_util(bld, util):
         obj.use    += ' libappleutility'
     elif bld.env['build_target'] == 'mingw':
         obj.linkflags += ' -mwindows'
-    else:
-        obj.linkflags += ' -zexecstack'
 
     obj.includes += ['../libs']
 

wscript

@@ -80,6 +80,8 @@ compiler_flags_dictionaries= {
         'pic': '-fPIC',
         # Flags required to compile C code with anonymous unions (only part of C11)
         'c-anonymous-union': '-fms-extensions',
+        # optional -zexecstack linkflag
+        'execstack': '',
     },
     'msvc' : {
         'debuggable' : ['/DDEBUG', '/Od', '/Zi', '/MDd', '/Gd', '/EHsc'],
@@ -113,6 +115,7 @@ compiler_flags_dictionaries= {
         'neon': '',
         'pic': '',
         'c-anonymous-union': '',
+        'execstack': '',
     },
 }
 
@@ -755,6 +758,13 @@ int main() { return 0; }''',
     # Do not use Boost.System library
     cxx_flags.append('-DBOOST_ERROR_CODE_HEADER_ONLY')
 
+    if platform == 'linux' and not conf.options.no_execstack:
+        if conf.check_cxx(linkflags=["-zexecstack"], mandatory = False, execute = False, msg = 'Checking for gcc/lld-style -zexecstack'):
+            flags_dict['execstack'] = "-zexecstack"
+        elif conf.check_cxx(linkflags=["-z execstack"], mandatory = False, execute = False, msg = 'Checking for clang execstack'):
+            flags_dict['execstack'] = "-z execstack"
+
+
     # use sparingly, prefer runtime profile
     if Options.options.program_name.lower().startswith('mixbus'):
         compiler_flags.append ('-DMIXBUS')
@@ -912,6 +922,8 @@ def options(opt):
                     help='Enable support to import PTS/PTF/PTX sessions')
     opt.add_option('--no-threaded-waveviews', action='store_true', default=False, dest='no_threaded_waveviews',
                     help='Disable threaded waveview rendering')
+    opt.add_option('--no-execstack', action='store_true', default=False, dest='no_execstack',
+                    help='Disable executable stack (may break some plugins)')
     opt.add_option('--no-futex-semaphore', action='store_true', default=False, dest='no_futex_semaphore',
                     help='Disable use of futex for semaphores (Linux only)')
     opt.add_option(