From 91cd71f1a4b16ce370578bfd25c27909b0b20b2b Mon Sep 17 00:00:00 2001 From: Marshall Greenblatt Date: Wed, 12 Oct 2011 06:36:51 +0000 Subject: [PATCH] Add the CEF Automation Tool (issue #373). git-svn-id: https://chromiumembedded.googlecode.com/svn/trunk@310 5089003a-bbd8-11dd-ad1f-f1f9622dbc98 --- CHROMIUM_BUILD_COMPATIBILITY.txt | 4 +- tools/automate/automate.README.txt | 99 ++++++++++ tools/automate/automate.py | 280 +++++++++++++++++++++++++++++ 3 files changed, 381 insertions(+), 2 deletions(-) create mode 100644 tools/automate/automate.README.txt create mode 100644 tools/automate/automate.py diff --git a/CHROMIUM_BUILD_COMPATIBILITY.txt b/CHROMIUM_BUILD_COMPATIBILITY.txt index 68060ca6b..f3ffdd956 100644 --- a/CHROMIUM_BUILD_COMPATIBILITY.txt +++ b/CHROMIUM_BUILD_COMPATIBILITY.txt @@ -9,8 +9,8 @@ # To update to a specific Chromium revision: # gclient sync --revision src@chromium_revision --jobs 8 --force # -# This file is integrated with the CEF auto_build tool. See the README.txt -# file in the auto_build directory for more information. +# This file is integrated with the CEF Automation Tool. See the +# tools/automate/automate.README.txt file for more information. # # For general gclient usage information visit: # http://dev.chromium.org/developers/how-tos/get-the-code diff --git a/tools/automate/automate.README.txt b/tools/automate/automate.README.txt new file mode 100644 index 000000000..5683cd8a6 --- /dev/null +++ b/tools/automate/automate.README.txt @@ -0,0 +1,99 @@ +Chromium Embedded Framework (CEF) Automation Tool +------------------------------------------------------------------------------- + +Document Last Updated: October 10, 2011 + + +OVERVIEW +-------- + +The CEF Automation Tool can perform the following actions in an automated manner: + +1. Download/update the Chromium and CEF source code. +2. Build the CEF Debug and Release targets. +3. Create the CEF binary distribution package. + + +SETUP +----- + +1. Install Subversion [1] and Python [2]. Make sure the bin directories for both + programs are discoverable via your system PATH configuration. On Windows + install CppDoc [3] in the default location (assumes a 64bit version of + Windows 7). + +2. Install build tools. On Windows install Visual Studio 2008 or newer and all + required prerequisite software [4]. On Mac install Xcode 3.2 or newer. + +3. Configure the GYP environment. On Windows set the GYP_MSVS_VERSION + environment variable to "2008" or "2010" depending on which version of Visual + Studio you're building with. On Mac Lion set the GYP_DEFINES environment + variable to 'mac_sdk=10.6'. + +4. Checkout the "automate" folder to a location on your hard drive. For the + trunk version of CEF you can use the following command: + + svn checkout http://chromiumembedded.googlecode.com/svn/trunk/tools/automate /path/to/automate + +5. Run the automate.py script at whatever interval is appropriate (for each + CEF commit, once per day, once per week, etc): + + python /path/to/automate/automate.py --download-dir=/path/to/download + + +HOW IT WORKS +------------ + +The automate.py script performs the following actions in the download directory +specified by the "--download-dir" flag. This path value must contain no spaces. + +1. Retrieve the Chromium URL and revision associated with a CEF URL and + revision. A specific CEF URL and revision can be specified using the "--url" + and "--revision" flags. Otherwise, the current CEF trunk URL [7] and HEAD + revision will be used. The Chromium URL and revision are retrieved by + querying the SVN repository for a CHROMIUM_BUILD_COMPATIBILITY.txt file. + +2. If a "depot_tools" folder does not already exist download depot_tools [5]. + The depot_tools folder will be added to the beginning of the PATH to support + execution of the below steps. + +3. If a "chromium" folder does not already exist create it and configure + gclient [6] using the Chromium URL retrieved in step 1. To force execution of + this step use the "--force-config" flag. + +4. If the "--force-clean" flag is specified all Chromium changes will be + reverted, all unversioned files including the CEF folder will be deleted and + steps 6-10 will be triggered automatically. + +5. If both Chromium and CEF are already at the correct URL and revision the + script will exit at this point. To force execution of steps 6-8 use the + "--force-update" flag. To force execution of step 9 use the "--force-build" + flag. To force execution of step 10 use the "--force-distrib" flag. + +6. Use gclient [6] to update the Chromium source code to the correct revision. + +7. If a "chromium/src/cef" folder does not already exist download the CEF source + code at the correct revision. Otherwise, update CEF source code to the + correct revision. + +8. Run the "chromium/src/cef/cef_create_projects.[sh|bat]" script to apply CEF + patches and generate CEF project files. + +9. Run the "chromium/src/cef/tools/build_projects.[sh|bat]" script twice; once + to build the Debug target and once to build the Release target. + +10.Run the "chromium/src/cef/tools/make_distrib.[sh|bat]" script to build the + binary distribution package. The resulting package will be output in a + "chromium/src/cef/binary_distrib/cef_binary_rXXX_[windows|macosx]" folder. + + +REFERENCES +---------- + +[1] http://subversion.apache.org/ +[2] http://www.python.org/ +[3] http://www.cppdoc.com/ +[4] http://dev.chromium.org/developers/how-tos/build-instructions-windows +[5] http://dev.chromium.org/developers/how-tos/depottools +[6] http://dev.chromium.org/developers/how-tos/get-the-code +[7] http://chromiumembedded.googlecode.com/svn/trunk diff --git a/tools/automate/automate.py b/tools/automate/automate.py new file mode 100644 index 000000000..43800d3f9 --- /dev/null +++ b/tools/automate/automate.py @@ -0,0 +1,280 @@ +# Copyright (c) 2011 The Chromium Embedded Framework Authors. All rights +# reserved. Use of this source code is governed by a BSD-style license that +# can be found in the LICENSE file. + +from optparse import OptionParser +import os +import re +import shlex +import shutil +import subprocess +import sys +import urllib + +# default URL values +cef_url = 'http://chromiumembedded.googlecode.com/svn/trunk' +depot_tools_url = 'http://src.chromium.org/svn/trunk/tools/depot_tools' + +def run(command_line, working_dir, depot_tools_dir=None): + # add depot_tools to the path + env = os.environ + if not depot_tools_dir is None: + env['PATH'] = depot_tools_dir+os.pathsep+env['PATH'] + + sys.stdout.write('-------- Running "'+command_line+'" in "'+\ + working_dir+'"...'+"\n") + args = shlex.split(command_line.replace('\\', '\\\\')) + return subprocess.check_call(args, cwd=working_dir, env=env, shell=True) + +def get_svn_info(path): + """ Retrieves the URL and revision from svn info. """ + url = None; + rev = None; + if path[0:4] == 'http' or os.path.exists(path): + try: + stream = os.popen('svn info '+path); + for line in stream: + if line[0:4] == "URL:": + url = line[5:-1]; + elif line[0:9] == "Revision:": + rev = line[10:-1]; + except IOError, (errno, strerror): + sys.stderr.write('Failed to read revision from "svn info": ' + strerror) + raise + return {'url': url, 'revision': rev} + +# cannot be loaded as a module +if __name__ != "__main__": + sys.stderr.write('This file cannot be loaded as a module!') + sys.exit() + +# parse command-line options +disc = """ +This utility implements automation for the download, update, build and +distribution of CEF. +""" + +parser = OptionParser(description=disc) +parser.add_option('--download-dir', dest='downloaddir', metavar='DIR', + help='download directory with no spaces [required]') +parser.add_option('--revision', dest='revision', type="int", + help='CEF source revision') +parser.add_option('--url', dest='url', + help='CEF source URL') +parser.add_option('--force-config', + action='store_true', dest='forceconfig', default=False, + help='force Chromium configuration') +parser.add_option('--force-clean', + action='store_true', dest='forceclean', default=False, + help='force revert of all Chromium changes, deletion of '+\ + 'all unversioned files including the CEF folder and '+\ + 'trigger the force-update, force-build and '+\ + 'force-distrib options') +parser.add_option('--force-update', + action='store_true', dest='forceupdate', default=False, + help='force Chromium and CEF update') +parser.add_option('--force-build', + action='store_true', dest='forcebuild', default=False, + help='force CEF debug and release builds') +parser.add_option('--force-distrib', + action='store_true', dest='forcedistrib', default=False, + help='force creation of CEF binary distribution') +parser.add_option('--no-debug-build', + action='store_true', dest='nodebugbuild', default=False, + help="don't perform the CEF debug build") +parser.add_option('--no-release-build', + action='store_true', dest='noreleasebuild', default=False, + help="don't perform the CEF release build") +parser.add_option('--no-distrib', + action='store_true', dest='nodistrib', default=False, + help="don't create the CEF binary distribution") +(options, args) = parser.parse_args() + +# the downloaddir option is required +if options.downloaddir is None: + parser.print_help(sys.stderr) + sys.exit() + +# script directory +script_dir = os.path.dirname(__file__) + +if not options.url is None: + # set the CEF URL + cef_url = options.url + +if not options.revision is None: + # set the CEF revision + cef_rev = str(options.revision) +else: + # retrieve the CEF revision from the remote repo + info = get_svn_info(cef_url) + cef_rev = info['revision'] + +# Retrieve the Chromium URL and revision from the CEF repo +compat_url = cef_url + "/CHROMIUM_BUILD_COMPATIBILITY.txt?r="+cef_rev + +try: + # Read the remote URL contents + handle = urllib.urlopen(compat_url) + compat_value = handle.read().strip() + handle.close() + + # Parse the contents + config = eval(compat_value, {'__builtins__': None}, None) + + if not 'chromium_url' in config: + raise Exception("Missing chromium_url value") + if not 'chromium_revision' in config: + raise Exception("Missing chromium_revision value") + + chromium_url = config['chromium_url'] + chromium_rev = config['chromium_revision'] +except Exception, e: + sys.stderr.write('Failed to read URL and revision information from '+ \ + compat_url+"\n") + raise + +download_dir = options.downloaddir +if not os.path.exists(download_dir): + # create the download directory + os.makedirs(download_dir) + +# check the operating system +platform = ''; +script_ext = ''; +if sys.platform == 'win32': + platform = 'windows' + script_ext = '.bat' +elif sys.platform == 'darwin': + platform = 'macosx' + script_ext = '.sh' +elif sys.platform == 'linux2': + platform = 'linux' + script_ext = '.sh' + +# check if the "depot_tools" directory exists +depot_tools_dir = os.path.join(download_dir, 'depot_tools') +if not os.path.exists(depot_tools_dir): + # checkout depot_tools + run('svn checkout '+depot_tools_url+' '+depot_tools_dir, download_dir); + +# check if the "chromium" directory exists +chromium_dir = os.path.join(download_dir, 'chromium') +if not os.path.exists(chromium_dir): + # create the "chromium" directory + os.makedirs(chromium_dir) + +chromium_src_dir = os.path.join(chromium_dir, 'src') +cef_src_dir = os.path.join(chromium_src_dir, 'cef') +cef_tools_dir = os.path.join(cef_src_dir, 'tools') + +# retrieve the current CEF URL and revision +info = get_svn_info(cef_src_dir) +current_cef_url = info['url'] +current_cef_rev = info['revision'] + +# retrieve the current Chromium URL and revision +info = get_svn_info(chromium_src_dir) +current_chromium_url = info['url'] +current_chromium_rev = info['revision'] + +# test if the CEF URL changed +cef_url_changed = current_cef_url != cef_url +sys.stdout.write('CEF URL: '+current_cef_url+"\n") +if cef_url_changed: + sys.stdout.write(' -> CHANGED TO: '+cef_url+"\n") + +# test if the CEF revision changed +cef_rev_changed = current_cef_rev != cef_rev +sys.stdout.write('CEF Revision: '+current_cef_rev+"\n") +if cef_rev_changed: + sys.stdout.write(' -> CHANGED TO: '+cef_rev+"\n") + +# test if the Chromium URL changed +chromium_url_changed = current_chromium_url != chromium_url +sys.stdout.write('Chromium URL: '+current_chromium_url+"\n") +if chromium_url_changed: + sys.stdout.write(' -> CHANGED TO: '+chromium_url+"\n") + +# test if the Chromium revision changed +chromium_rev_changed = current_chromium_rev != chromium_rev +sys.stdout.write('Chromium Revision: '+current_chromium_rev+"\n") +if chromium_rev_changed: + sys.stdout.write(' -> CHANGED TO: '+chromium_rev+"\n") + +# true if anything changed +any_changed = chromium_url_changed or chromium_rev_changed or \ + cef_url_changed or cef_rev_changed +if not any_changed: + sys.stdout.write("No changes.\n") + +if chromium_url_changed or options.forceconfig: + # run gclient config to create the .gclient file + run('gclient config '+chromium_url, chromium_dir, depot_tools_dir) + + path = os.path.join(chromium_dir, '.gclient') + if not os.path.exists(path): + sys.stderr.write(".gclient file was not created\n") + raise Exception('.gclient file was not created') + + # read the resulting .gclient file + fp = open(path, 'r') + data = fp.read() + fp.close() + + # populate "custom_deps" section + data = data.replace('"custom_deps" : {', '"custom_deps" : {'+\ + "\n "+'"src/third_party/WebKit/LayoutTests": None,'+\ + "\n "+'"src/chrome_frame/tools/test/reference_build/chrome": None,'+\ + "\n "+'"src/chrome/tools/test/reference_build/chrome_mac": None,'+\ + "\n "+'"src/chrome/tools/test/reference_build/chrome_win": None,'+\ + "\n "+'"src/chrome/tools/test/reference_build/chrome_linux": None,') + + # write the new .gclient file + fp = open(path, 'w') + fp.write(data) + fp.close() + +if options.forceclean: + if os.path.exists(chromium_src_dir): + # revert all Chromium changes and delete all unversioned files + run('gclient revert -n', chromium_dir, depot_tools_dir) + + # force update, build and distrib steps + options.forceupdate = True + options.forcebuild = True + options.forcedistrib = True + +if chromium_url_changed or chromium_rev_changed or options.forceupdate: + # download/update the Chromium source code + run('gclient sync --revision src@'+chromium_rev+' --jobs 8 --force', \ + chromium_dir, depot_tools_dir) + +if not os.path.exists(cef_src_dir) or cef_url_changed: + if cef_url_changed and os.path.exists(cef_src_dir): + # delete the cef directory (it will be re-downloaded) + shutil.rmtree(cef_src_dir) + + # download the CEF source code + run('svn checkout '+cef_url+' -r '+cef_rev+' '+cef_src_dir, download_dir) +elif cef_rev_changed or options.forceupdate: + # update the CEF source code + run('svn update -r '+cef_rev+' '+cef_src_dir, download_dir) + +if any_changed or options.forceupdate: + # create CEF projects + run('cef_create_projects'+script_ext, cef_src_dir, depot_tools_dir) + +if any_changed or options.forcebuild: + if not options.nodebugbuild: + # make CEF Debug build + run('build_projects'+script_ext+' Debug', cef_tools_dir, depot_tools_dir) + + if not options.noreleasebuild: + # make CEF Release build + run('build_projects'+script_ext+' Release', cef_tools_dir, depot_tools_dir) + +if any_changed or options.forcedistrib: + if not options.nodistrib: + # make CEF binary distribution + run('make_distrib'+script_ext, cef_tools_dir, depot_tools_dir)