From be69bf9eb157a42db0fb6a59729e49375e1f5272 Mon Sep 17 00:00:00 2001 From: Marshall Greenblatt Date: Tue, 10 Sep 2013 20:02:09 +0000 Subject: [PATCH] Enhancements to automate.py and make_distrib.py (issue #1072): - Update the depot tools checkout if the checkout is managed by automate.py (on windows it's mandatory to run depot tools update to install svn, git and python). - Add "--no-update" switch to allow users to run only build and distrib steps without updating CEF and Chromium. This is useful when some custom actions have to be done between the checkout and build steps. - Add "--build-tests" switch to build the cef_unittests target in addition to the cefclient target. - Add "--verbose" switch to have ninja output more details while building. - Add "--build-log-file" switch to write build logs to a file on disk. - Add 7z compression support based on the presence of a CEF_COMMAND_7ZIP environment variable. - Create distrib folders using the architecture (32/64) string. git-svn-id: https://chromiumembedded.googlecode.com/svn/trunk@1434 5089003a-bbd8-11dd-ad1f-f1f9622dbc98 --- tools/automate/automate.py | 80 +++++++++++++++++++++++++++++++------- tools/make_distrib.py | 46 ++++++++++++++++++---- 2 files changed, 104 insertions(+), 22 deletions(-) diff --git a/tools/automate/automate.py b/tools/automate/automate.py index 47f9d2b3c..2df8d202d 100644 --- a/tools/automate/automate.py +++ b/tools/automate/automate.py @@ -19,18 +19,24 @@ import zipfile cef_url = 'http://chromiumembedded.googlecode.com/svn/trunk/cef3' depot_tools_url = 'http://src.chromium.org/svn/trunk/tools/depot_tools' -def run(command_line, working_dir, depot_tools_dir=None): +def run(command_line, working_dir, depot_tools_dir=None, output_file=None): # add depot_tools to the path env = os.environ if not depot_tools_dir is None: env['PATH'] = depot_tools_dir+os.pathsep+env['PATH'] - + sys.stdout.write('-------- Running "'+command_line+'" in "'+\ working_dir+'"...'+"\n") if not options.dryrun: args = shlex.split(command_line.replace('\\', '\\\\')) - return subprocess.check_call(args, cwd=working_dir, env=env, + + if not output_file: + return subprocess.check_call(args, cwd=working_dir, env=env, shell=(sys.platform == 'win32')) + with open(output_file, "w") as f: + return subprocess.check_call(args, cwd=working_dir, env=env, + shell=(sys.platform == 'win32'), + stderr=subprocess.STDOUT, stdout=f) def check_url(url): """ Check the URL and raise an exception if invalid. """ @@ -129,7 +135,7 @@ def onerror(func, path, exc_info): func(path) else: raise - + # cannot be loaded as a module if __name__ != "__main__": sys.stderr.write('This file cannot be loaded as a module!') @@ -165,9 +171,16 @@ parser.add_option('--force-clean', parser.add_option('--force-update', action='store_true', dest='forceupdate', default=False, help='force Chromium and CEF update') +parser.add_option('--no-update', + action='store_true', dest='noupdate', default=False, + help='do not update Chromium and CEF.' +\ + 'Cannot be used along with --force[update|config|clean]') parser.add_option('--force-build', action='store_true', dest='forcebuild', default=False, help='force CEF debug and release builds') +parser.add_option('--build-tests', + action='store_true', dest='buildtests', default=False, + help='build cef_unittests target besides cefclient') parser.add_option('--force-distrib', action='store_true', dest='forcedistrib', default=False, help='force creation of CEF binary distribution') @@ -201,6 +214,12 @@ parser.add_option('--no-distrib-archive', parser.add_option('--ninja-build', action='store_true', dest='ninjabuild', default=False, help="build using ninja") +parser.add_option('--verbose', + action='store_true', dest='verbose', default=False, + help='show all command lines while building') +parser.add_option('--build-log-file', + action='store_true', dest='buildlogfile', default=False, + help='write build logs to files') parser.add_option('--x64-build', action='store_true', dest='x64build', default=False, help='build for 64-bit systems (Windows and Mac OS X only)') @@ -233,6 +252,11 @@ if (options.noreleasebuild and (options.minimaldistrib or options.minimaldistrib parser.print_help(sys.stderr) sys.exit() +if options.noupdate and (options.forceclean or options.forceupdate or options.forceconfig): + print "Invalid combination of options." + print "--no-update cannot be used along with --force-[update|config|clean]\n" + sys.exit() + if options.x64build and platform != 'windows' and platform != 'macosx': print 'The x64 build option is only used on Windows and Mac OS X.' sys.exit() @@ -269,6 +293,13 @@ if not os.path.exists(depot_tools_dir): # checkout depot_tools run('svn checkout '+depot_tools_url+' '+depot_tools_dir, download_dir) +if not options.noupdate and options.depottools == '': + # Update depot_tools. It will download required scripts (svn, python, ...) + if sys.platform == 'win32': + run('update_depot_tools.bat', depot_tools_dir); + else: + run('update_depot_tools', depot_tools_dir); + if sys.platform == 'win32': # Force use of the SVN version bundled with depot_tools. svn_exe = os.path.join(depot_tools_dir, 'svn.bat') @@ -344,17 +375,21 @@ if release_url is None: current_chromium_url = info['url'] current_chromium_rev = info['revision'] +changed_to_message = ' -> CHANGED TO: ' +if options.noupdate: + changed_to_message = ' -> AVAILABLE: ' + # test if the CEF URL changed cef_url_changed = current_cef_url != cef_url sys.stdout.write('CEF URL: '+current_cef_url+"\n") if cef_url_changed: - sys.stdout.write(' -> CHANGED TO: '+cef_url+"\n") + sys.stdout.write(changed_to_message+cef_url+"\n") # test if the CEF revision changed cef_rev_changed = current_cef_rev != cef_rev sys.stdout.write('CEF Revision: '+current_cef_rev+"\n") if cef_rev_changed: - sys.stdout.write(' -> CHANGED TO: '+cef_rev+"\n") + sys.stdout.write(changed_to_message+cef_rev+"\n") release_url_changed = False chromium_url_changed = False @@ -365,13 +400,13 @@ if release_url is None: chromium_url_changed = current_chromium_url != chromium_url sys.stdout.write('Chromium URL: '+current_chromium_url+"\n") if chromium_url_changed: - sys.stdout.write(' -> CHANGED TO: '+chromium_url+"\n") + sys.stdout.write(changed_to_message+chromium_url+"\n") # test if the Chromium revision changed chromium_rev_changed = current_chromium_rev != chromium_rev sys.stdout.write('Chromium Revision: '+current_chromium_rev+"\n") if chromium_rev_changed: - sys.stdout.write(' -> CHANGED TO: '+chromium_rev+"\n") + sys.stdout.write(changed_to_message+chromium_rev+"\n") else: # test if the release URL changed current_release_url = 'None' @@ -395,14 +430,23 @@ else: release_url_changed = current_release_url != release_url sys.stdout.write('Release URL: '+current_release_url+"\n") if release_url_changed: - sys.stdout.write(' -> CHANGED TO: '+release_url+"\n") + sys.stdout.write(changed_to_message+release_url+"\n") # true if anything changed any_changed = release_url_changed or chromium_url_changed or \ chromium_rev_changed or cef_url_changed or cef_rev_changed if not any_changed: sys.stdout.write("No changes.\n") - +elif options.noupdate: + sys.stdout.write("You have updates. Remove --no-update flag to update source code\n") + release_url_changed = False + chromium_url_changed = False + chromium_rev_changed = False + cef_url_changed = False + cef_rev_changed = False + any_changed = False + + if release_url_changed or chromium_url_changed or options.forceconfig: if release_url is None: url = chromium_url @@ -507,7 +551,11 @@ if any_changed or options.forceupdate: if any_changed or options.forcebuild: if options.ninjabuild: command = 'ninja -C ' + if options.verbose: + command = 'ninja -v -C' target = ' cefclient' + if options.buildtests: + target = ' cefclient cef_unittests' build_dir_suffix = '' if platform == 'windows' and options.x64build: build_dir_suffix = '_x64' @@ -515,22 +563,26 @@ if any_changed or options.forcebuild: if not options.nodebugbuild: # make CEF Debug build run(command + os.path.join('out', 'Debug' + build_dir_suffix) + target, \ - chromium_src_dir, depot_tools_dir) + chromium_src_dir, depot_tools_dir, + os.path.join(chromium_src_dir, 'ninja-build-debug.log')) if not options.noreleasebuild: # make CEF Release build run(command + os.path.join('out', 'Release' + build_dir_suffix) + target, \ - chromium_src_dir, depot_tools_dir) + chromium_src_dir, depot_tools_dir, + os.path.join(chromium_src_dir, 'ninja-build-release.log')) else: path = os.path.join(cef_tools_dir, 'build_projects'+script_ext) if not options.nodebugbuild: # make CEF Debug build - run(path+' Debug', cef_tools_dir, depot_tools_dir) + run(path+' Debug', cef_tools_dir, depot_tools_dir, + os.path.join(chromium_src_dir, 'build-debug.log')) if not options.noreleasebuild: # make CEF Release build - run(path+' Release', cef_tools_dir, depot_tools_dir) + run(path+' Release', cef_tools_dir, depot_tools_dir, + os.path.join(chromium_src_dir, 'build-release.log')) if (any_changed or options.forcedistrib) and not options.nodistrib: if not options.forceclean and options.cleanartifacts: diff --git a/tools/make_distrib.py b/tools/make_distrib.py index 1156580c7..fc745ba01 100644 --- a/tools/make_distrib.py +++ b/tools/make_distrib.py @@ -28,6 +28,11 @@ def create_archive(input_dir, zip_file): addDir(input_dir) zf.close() +def create_7z_archive(input_dir, zip_file): + """ Creates a 7z archive of the specified input directory. """ + command = os.environ['CEF_COMMAND_7ZIP'] + run('"' + command + '" a -y ' + zip_file + ' ' + input_dir, os.path.split(zip_file)[0]) + def create_output_dir(name, parent_dir): """ Creates an output directory and adds the path to the archive list. """ output_dir = os.path.abspath(os.path.join(parent_dir, name)) @@ -112,7 +117,7 @@ def create_readme(): def eval_file(src): """ Loads and evaluates the contents of the specified file. """ return eval(read_file(src), {'__builtins__': None}, None) - + def transfer_gypi_files(src_dir, gypi_paths, gypi_path_prefix, dst_dir, quiet): """ Transfer files from one location to another. """ for path in gypi_paths: @@ -137,24 +142,24 @@ def transfer_files(cef_dir, script_dir, transfer_cfg, output_dir, quiet): """ Transfer files based on the specified configuration. """ if not path_exists(transfer_cfg): return - + configs = eval_file(transfer_cfg) for cfg in configs: dst = os.path.join(output_dir, cfg['target']) - + # perform a copy if source is specified if not cfg['source'] is None: src = os.path.join(cef_dir, cfg['source']) dst_path = os.path.dirname(dst) make_dir(dst_path, quiet) copy_file(src, dst, quiet) - + # place a readme file in the destination directory readme = os.path.join(dst_path, 'README-TRANSFER.txt') if not path_exists(readme): copy_file(os.path.join(script_dir, 'distrib/README-TRANSFER.txt'), readme) open(readme, 'ab').write(cfg['source']+"\n") - + # perform any required post-processing if 'post-process' in cfg: post = cfg['post-process'] @@ -357,9 +362,28 @@ chromium_ver = args['MAJOR']+'.'+args['MINOR']+'.'+args['BUILD']+'.'+args['PATCH # list of output directories to be archived archive_dirs = [] +platform_arch = '32' +if options.x64build: + platform_arch = '64' + +if platform == 'linux': + platform_arch = '' + lib_dir_name = 'lib.target' + if options.ninjabuild: + lib_dir_name = 'lib' + release_libcef_path = os.path.join(src_dir, 'out', 'Release', lib_dir_name, 'libcef.so'); + debug_libcef_path = os.path.join(src_dir, 'out', 'Debug', lib_dir_name, 'libcef.so'); + file_desc = '' + output = subprocess.check_output('file ' + release_libcef_path + ' ' + debug_libcef_path + '; exit 0', + env=os.environ, stderr=subprocess.STDOUT, shell=True) + if output.find('32-bit') != -1: + platform_arch = '32' + if output.find('64-bit') != -1: + platform_arch = '64' + # output directory output_dir_base = 'cef_binary_' + cef_ver -output_dir_name = output_dir_base + '_' + platform +output_dir_name = output_dir_base + '_' + platform + platform_arch if options.minimal: mode = 'minimal' @@ -676,8 +700,14 @@ elif platform == 'linux': if not options.noarchive: # create an archive for each output directory + archive_extenstion = '.zip' + if os.getenv('CEF_COMMAND_7ZIP', '') != '': + archive_extenstion = '.7z' for dir in archive_dirs: - zip_file = os.path.split(dir)[1] + '.zip' + zip_file = os.path.split(dir)[1] + archive_extenstion if not options.quiet: sys.stdout.write('Creating '+zip_file+"...\n") - create_archive(dir, os.path.join(dir, os.pardir, zip_file)) + if archive_extenstion == '.zip': + create_archive(dir, os.path.join(dir, os.pardir, zip_file)) + else: + create_7z_archive(dir, os.path.join(dir, os.pardir, zip_file))