From db2c35a57c118dd7c440b60352a9047e238d0b29 Mon Sep 17 00:00:00 2001 From: Brock York Date: Fri, 12 Jun 2020 16:16:09 +1000 Subject: [PATCH] SCons: Update SCons build scripts to support Python 3 and SCons 3 Add parentheses to print statements Use subprocess.check_output instead of commands.getstatusoutput to run external processes Use pickle instead of cPickle library This allows building on Linux distributions that have switched to python 3 by default but also retains backwards compatibility with python 2.7 --- SConscript.lib | 4 +--- SConstruct | 14 +++++------ config.py | 63 +++++++++++++++++++++++++++----------------------- utils.py | 39 ++++++++++++++++--------------- 4 files changed, 62 insertions(+), 58 deletions(-) diff --git a/SConscript.lib b/SConscript.lib index 0aaa81a4..f02ee6d4 100644 --- a/SConscript.lib +++ b/SConscript.lib @@ -9,7 +9,6 @@ Import( [ 'utils', 'config', 'settings', 'project' ] ) ( libpath, libname ) = os.path.split( project ) libname = os.path.splitext( libname )[0] - env = Environment( ENV = os.environ ) settings.SetupEnvironment( env, config['name'] ) proj = utils.vcxproj( os.path.join( GetLaunchDir(), project ) ) @@ -29,6 +28,5 @@ except: objects = [] for i in files + add_sources: - objects.append( emit_func( os.path.join( libpath, i ) ) ) - + objects.append( emit_func( source=os.path.join( libpath, i ) ) ) Return( 'objects' ) diff --git a/SConstruct b/SConstruct index 5bca1451..fe1fb1ac 100644 --- a/SConstruct +++ b/SConstruct @@ -3,7 +3,7 @@ # TTimo # http://scons.org/ -import sys, os, platform, cPickle +import sys, os, platform, pickle import utils, config @@ -43,10 +43,10 @@ active_configs = [] # load up configurations from the save file if ( os.path.exists( conf_filename ) ): f = open( conf_filename ) - print 'reading saved configuration from site.conf' + print( 'reading saved configuration from site.conf' ) try: while ( True ): - c = cPickle.load( f ) + c = pickle.load( f ) active_configs.append( c ) except: pass @@ -57,12 +57,12 @@ active_configs = config.ConfigParser().parseStatements( active_configs, config_s assert( len( active_configs ) >= 1 ) # save the config -print 'saving updated configuration' +print( 'saving updated configuration' ) f = open( conf_filename, 'wb' ) for c in active_configs: - cPickle.dump( c, f, -1 ) + pickle.dump( c, f, -1 ) -print 'emit build rules' +print( 'emit build rules' ) for c in active_configs: - print 'emit configuration: %s' % repr( c ) + print( 'emit configuration: %s' % repr( c ) ) c.emit() diff --git a/config.py b/config.py index 1ff26e32..a757268f 100644 --- a/config.py +++ b/config.py @@ -1,5 +1,5 @@ -import sys, os, traceback, platform, re, commands, platform, subprocess -import urllib2, zipfile, shutil, pprint +import sys, os, traceback, platform, re, subprocess, platform +import urllib3, zipfile, shutil, pprint if __name__ != '__main__': from SCons.Script import * @@ -218,13 +218,14 @@ class Config: def SetupEnvironment( self, env, config, useGtk = False, useGtkGL = False, useJPEG = False, useZ = False, usePNG = False ): env['CC'] = self.cc env['CXX'] = self.cxx - ( ret, xml2 ) = commands.getstatusoutput( 'xml2-config --cflags' ) - if ( ret != 0 ): - print 'xml2-config failed' + try: + xml2 = subprocess.check_output( ['xml2-config', '--cflags'] ).decode( 'utf-8' ) + except subprocess.CalledProcessError as cpe: + print( 'xml2-config failed with error code {} and output:{}'.format( cpe.returncode, cpe.output ) ) assert( False ) - xml2libs = commands.getoutput( 'xml2-config --libs' ) env.ParseConfig( 'xml2-config --libs' ) - baseflags = [ '-pipe', '-Wall', '-fmessage-length=0', '-fvisibility=hidden', xml2.split( ' ' ) ] + #Need to strip on xml2-config output. It has a stray \n and that completely screws up scons calling g++ + baseflags = [ '-pipe', '-Wall', '-fmessage-length=0', '-fvisibility=hidden', xml2.strip().split( ' ' ) ] if ( useGtk ): env.ParseConfig( 'pkg-config gtk+-2.0 --cflags --libs' ) @@ -409,24 +410,28 @@ class Config: print( 'Lookup and bundle the PNG and JPEG libraries' ) # radiant.bin doesn't link to jpeg lib directly, grab that from a module # Python 2.7 only! - #module_ldd = subprocess.check_output( 'ldd -r install/modules/image.so', shell = True ) - p = subprocess.Popen( 'ldd -r install/modules/image.so', shell = True, stdout = subprocess.PIPE ) - module_ldd = p.communicate()[0] -# print( module_ldd ) + try: + module_ldd = subprocess.check_output( ['ldd', '-r', 'install/modules/image.so'] ).decode( 'utf-8' ) + except subprocess.CalledProcessError as cpe: + print( 'ldd failed with error code {} and output:{}'.format(cpe.returncode, cpe.output )) + assert( False ) def find_library( output, libname ): - print output - print libname - match = filter( lambda l : l.find( libname ) != -1, output.split( '\n' ) )[0] - return re.split( '.*=> (.*) .*', match )[1] + for line in output.split( '\n' ): + if libname in line: + return re.split( '.*=> (.*) .*', line)[1] + return "" jpeg_path = find_library( module_ldd, 'libjpeg' ) print( 'JPEG library: %s' % repr( jpeg_path ) ) - p = subprocess.Popen( 'ldd -r install/modules/imagepng.so', shell = True, stdout = subprocess.PIPE ) - module_ldd = p.communicate()[0] - + try: + module_ldd = subprocess.check_output( ['ldd', '-r', 'install/modules/imagepng.so'] ).decode( 'utf-8' ) + except subprocess.CalledProcessError as cpe: + print( 'ldd failed with error code {} and output:{}'.format(cpe.returncode, cpe.output )) + assert( False ) png_path = find_library( module_ldd, 'libpng' ) + print( 'PNG library: %s' % repr( png_path ) ) shutil.copy( jpeg_path, 'install' ) @@ -460,16 +465,16 @@ class ConfigParser: statement_re = re.compile( '(.*)=(.*)' ) value_list_re = re.compile( '([^,]*),?' ) if ( not statement_re.match( s ) ): - print 'syntax error (statement match): %s' % repr( s ) + print( 'syntax error (statement match): %s' % repr( s ) ) return statement_split = statement_re.split( s ) if ( len( statement_split ) != 4 ): - print 'syntax error (statement split): %s' % repr( s ) + print( 'syntax error (statement split): %s' % repr( s ) ) return ( foo, name, value, bar ) = statement_split value_split = value_list_re.split( value ) if ( len( value_split ) < 2 or len( value_split ) % 2 != 1 ): - print 'syntax error (value split): %s' % ( repr( value_split ) ) + print( 'syntax error (value split): %s' % ( repr( value_split ) ) ) return try: value_array = [] @@ -479,8 +484,8 @@ class ConfigParser: value_array.append( value_split.pop() ) value_split.pop() except: - print traceback.print_exception( sys.exc_type, sys.exc_value, sys.exc_traceback ) - print 'syntax error (value to array): %s' % ( repr( value_split ) ) + print( traceback.print_exception( sys.exc_type, sys.exc_value, sys.exc_traceback ) ) + print( 'syntax error (value to array): %s' % ( repr( value_split ) ) ) return return ( name, value_array ) @@ -504,13 +509,13 @@ class ConfigParser: ret = self._parseStatement( s ) if ( ret is None ): - print 'stop statement parse at %s' % repr( s ) + print( 'stop statement parse at %s' % repr( s ) ) break ( name, value_array ) = ret try: processor = self.operators[name] except: - print 'unknown operator %s - stop statement parse at %s' % ( repr( name ), repr( s ) ) + print( 'unknown operator %s - stop statement parse at %s' % ( repr( name ), repr( s ) ) ) break processor( value_array ) @@ -518,7 +523,7 @@ class ConfigParser: self.configs.append( self.current_config ) # make sure there is at least one config if ( len( self.configs ) == 0 ): - print 'pushing a default config' + print( 'pushing a default config' ) self.configs.append( Config() ) return self.configs @@ -533,17 +538,17 @@ class TestConfigParse( unittest.TestCase ): # test basic config parsing # needs to cleanly stop at the first config statement that is not recognized configs = self.parser.parseStatements( None, [ 'game=missionpack', 'config=qvm', 'foobar' ] ) - print repr( configs ) + print( repr( configs ) ) def testMultiParse( self ): # multiple configs seperated by commas configs = self.parser.parseStatements( None, [ 'target=server,game,cgame' ] ) - print repr( configs ) + print( repr( configs ) ) def testOp( self ): # test the operator for multiple configs configs = self.parser.parseStatements( None, [ 'target=core', 'config=release', 'op=push', 'target=game,cgame,ui', 'config=debug' ] ) - print repr( configs ) + print( repr( configs ) ) if __name__ == '__main__': unittest.main() diff --git a/utils.py b/utils.py index 2b76bff5..0a9cd6d0 100644 --- a/utils.py +++ b/utils.py @@ -3,14 +3,14 @@ # TTimo # http://scons.org/ -import os, commands, platform, xml.sax, re, string, platform +import os, subprocess, platform, xml.sax, re, string, platform class vcxproj( xml.sax.handler.ContentHandler ): def __init__( self, filepath ): self.source_files = [] self.misc_files = [] self._files = [] - print 'parse %s' % filepath + print( 'parse %s' % filepath ) xml.sax.parse( filepath, self ) def getSourceFiles( self ): @@ -30,8 +30,8 @@ class vcxproj( xml.sax.handler.ContentHandler ): def startElement( self, name, attrs ): if ( name == 'ClCompile' ): - if ( attrs.has_key('Include') ): - self._files.append( attrs.getValue('Include') ) + if ( 'Include' in attrs.getNames() ): + self._files.append( attrs.getValue('Include') ) def endDocument( self ): # split into source and headers, remap path seperator to the platform @@ -39,10 +39,10 @@ class vcxproj( xml.sax.handler.ContentHandler ): if ( platform.system() != 'Windows' ): f = f.replace( '\\', '/' ) if ( f[-2:] == '.c' or f[-4:] == '.cpp' ): - self.source_files.append( f.encode('ascii') ) + self.source_files.append( f ) else: self.misc_files.append( f ) - print '%d source files' % len( self.source_files ) + print( '%d source files' % len( self.source_files ) ) # action uses LDD to verify that the source doesn't hold unresolved symbols # setup as an AddPostAction of a regular SharedLibrary call @@ -50,19 +50,20 @@ def CheckUnresolved( source, target, env ): # TODO: implement this for OSX if ( platform.system() == 'Darwin' ): return None - # TODO: implement this for FreeBSD - if ( platform.system() == 'FreeBSD' ): - return None - print 'CheckUnresolved %s' % target[0].abspath + # TODO: implement this for FreeBSD + if ( platform.system() == 'FreeBSD' ): + return None + print( 'CheckUnresolved %s' % target[0].abspath ) if ( not os.path.isfile( target[0].abspath ) ): - print 'CheckUnresolved: %s does not exist' % target[0] + print( 'CheckUnresolved: %s does not exist' % target[0] ) return 1 # fail - ( status, output ) = commands.getstatusoutput( 'ldd -r %s' % target[0] ) - if ( status != 0 ): - print 'CheckUnresolved: ldd command failed (exit code %d)' % status - os.system( 'rm %s' % target[ 0 ] ) - return 1 # fail - lines = string.split( output, '\n' ) + try: + stdout = subprocess.check_output( ['ldd', '-r', str(target[0])] ).decode( 'utf-8' ) + except subprocess.CalledProcessError as cpe: + print( 'CheckUnresolved: ldd command failed (exit code {})'.format( cpe.returncode ) ) + os.system( 'rm %s' % target[ 0 ] ) + return 1 # fail + lines = stdout.split( '\n' ) have_undef = 0 for i_line in lines: regex = re.compile('undefined symbol: (.*)\t\\((.*)\\)') @@ -73,8 +74,8 @@ def CheckUnresolved( source, target, env ): except: have_undef = 1 if ( have_undef ): - print output - print "CheckUnresolved: undefined symbols" + print( output ) + print( "CheckUnresolved: undefined symbols" ) os.system('rm %s' % target[0]) return 1