Add the CEF Automation Tool (issue #373).

git-svn-id: https://chromiumembedded.googlecode.com/svn/trunk@310 5089003a-bbd8-11dd-ad1f-f1f9622dbc98

CHROMIUM_BUILD_COMPATIBILITY.txt

@@ -9,8 +9,8 @@
 # To update to a specific Chromium revision:
 #   gclient sync --revision src@chromium_revision --jobs 8 --force
 #
-# This file is integrated with the CEF auto_build tool. See the README.txt
+# This file is integrated with the CEF Automation Tool. See the
-# file in the auto_build directory for more information.
+# tools/automate/automate.README.txt file for more information.
 #
 # For general gclient usage information visit:
 # http://dev.chromium.org/developers/how-tos/get-the-code

tools/automate/automate.README.txt

@@ -0,0 +1,99 @@
+Chromium Embedded Framework (CEF) Automation Tool
+-------------------------------------------------------------------------------
+
+Document Last Updated: October 10, 2011
+
+
+OVERVIEW
+--------
+
+The CEF Automation Tool can perform the following actions in an automated manner:
+
+1. Download/update the Chromium and CEF source code.
+2. Build the CEF Debug and Release targets.
+3. Create the CEF binary distribution package.
+
+
+SETUP
+-----
+
+1. Install Subversion [1] and Python [2]. Make sure the bin directories for both
+   programs are discoverable via your system PATH configuration. On Windows
+   install CppDoc [3] in the default location (assumes a 64bit version of
+   Windows 7).
+
+2. Install build tools. On Windows install Visual Studio 2008 or newer and all
+   required prerequisite software [4]. On Mac install Xcode 3.2 or newer.
+
+3. Configure the GYP environment. On Windows set the GYP_MSVS_VERSION
+   environment variable to "2008" or "2010" depending on which version of Visual
+   Studio you're building with. On Mac Lion set the GYP_DEFINES environment
+   variable to 'mac_sdk=10.6'.
+
+4. Checkout the "automate" folder to a location on your hard drive. For the
+   trunk version of CEF you can use the following command:
+
+   svn checkout http://chromiumembedded.googlecode.com/svn/trunk/tools/automate /path/to/automate
+
+5. Run the automate.py script at whatever interval is appropriate (for each
+   CEF commit, once per day, once per week, etc):
+
+   python /path/to/automate/automate.py --download-dir=/path/to/download
+
+
+HOW IT WORKS
+------------
+
+The automate.py script performs the following actions in the download directory
+specified by the "--download-dir" flag. This path value must contain no spaces.
+
+1. Retrieve the Chromium URL and revision associated with a CEF URL and
+   revision. A specific CEF URL and revision can be specified using the "--url"
+   and "--revision" flags. Otherwise, the current CEF trunk URL [7] and HEAD
+   revision will be used. The Chromium URL and revision are retrieved by
+   querying the SVN repository for a CHROMIUM_BUILD_COMPATIBILITY.txt file.
+
+2. If a "depot_tools" folder does not already exist download depot_tools [5].
+   The depot_tools folder will be added to the beginning of the PATH to support
+   execution of the below steps.
+
+3. If a "chromium" folder does not already exist create it and configure
+   gclient [6] using the Chromium URL retrieved in step 1. To force execution of
+   this step use the "--force-config" flag.
+
+4. If the "--force-clean" flag is specified all Chromium changes will be
+   reverted, all unversioned files including the CEF folder will be deleted and
+   steps 6-10 will be triggered automatically.
+
+5. If both Chromium and CEF are already at the correct URL and revision the
+   script will exit at this point. To force execution of steps 6-8 use the
+   "--force-update" flag. To force execution of step 9 use the "--force-build"
+   flag. To force execution of step 10 use the "--force-distrib" flag.
+
+6. Use gclient [6] to update the Chromium source code to the correct revision.
+
+7. If a "chromium/src/cef" folder does not already exist download the CEF source
+   code at the correct revision. Otherwise, update CEF source code to the
+   correct revision.
+
+8. Run the "chromium/src/cef/cef_create_projects.[sh|bat]" script to apply CEF
+   patches and generate CEF project files.
+
+9. Run the "chromium/src/cef/tools/build_projects.[sh|bat]" script twice; once
+   to build the Debug target and once to build the Release target.
+
+10.Run the "chromium/src/cef/tools/make_distrib.[sh|bat]" script to build the
+   binary distribution package. The resulting package will be output in a
+   "chromium/src/cef/binary_distrib/cef_binary_rXXX_[windows|macosx]" folder.
+
+
+REFERENCES
+----------
+
+[1] http://subversion.apache.org/
+[2] http://www.python.org/
+[3] http://www.cppdoc.com/
+[4] http://dev.chromium.org/developers/how-tos/build-instructions-windows
+[5] http://dev.chromium.org/developers/how-tos/depottools
+[6] http://dev.chromium.org/developers/how-tos/get-the-code
+[7] http://chromiumembedded.googlecode.com/svn/trunk

tools/automate/automate.py

@@ -0,0 +1,280 @@
+# Copyright (c) 2011 The Chromium Embedded Framework Authors. All rights
+# reserved. Use of this source code is governed by a BSD-style license that
+# can be found in the LICENSE file.
+
+from optparse import OptionParser
+import os
+import re
+import shlex
+import shutil
+import subprocess
+import sys
+import urllib
+
+# default URL values
+cef_url = 'http://chromiumembedded.googlecode.com/svn/trunk'
+depot_tools_url = 'http://src.chromium.org/svn/trunk/tools/depot_tools'
+
+def run(command_line, working_dir, depot_tools_dir=None):
+  # add depot_tools to the path
+  env = os.environ
+  if not depot_tools_dir is None:
+    env['PATH'] = depot_tools_dir+os.pathsep+env['PATH']
+  
+  sys.stdout.write('-------- Running "'+command_line+'" in "'+\
+                   working_dir+'"...'+"\n")
+  args = shlex.split(command_line.replace('\\', '\\\\'))
+  return subprocess.check_call(args, cwd=working_dir, env=env, shell=True)
+
+def get_svn_info(path):
+  """ Retrieves the URL and revision from svn info. """
+  url = None;
+  rev = None;
+  if path[0:4] == 'http' or os.path.exists(path):
+    try:
+      stream = os.popen('svn info '+path);
+      for line in stream:
+        if line[0:4] == "URL:":
+          url = line[5:-1];
+        elif line[0:9] == "Revision:":
+          rev = line[10:-1];
+    except IOError, (errno, strerror):
+      sys.stderr.write('Failed to read revision from "svn info": ' + strerror)
+      raise
+  return {'url': url, 'revision': rev}
+  
+# cannot be loaded as a module
+if __name__ != "__main__":
+  sys.stderr.write('This file cannot be loaded as a module!')
+  sys.exit()
+
+# parse command-line options
+disc = """
+This utility implements automation for the download, update, build and
+distribution of CEF.
+"""
+
+parser = OptionParser(description=disc)
+parser.add_option('--download-dir', dest='downloaddir', metavar='DIR',
+                  help='download directory with no spaces [required]')
+parser.add_option('--revision', dest='revision', type="int",
+                  help='CEF source revision')
+parser.add_option('--url', dest='url',
+                  help='CEF source URL')
+parser.add_option('--force-config',
+                  action='store_true', dest='forceconfig', default=False,
+                  help='force Chromium configuration')
+parser.add_option('--force-clean',
+                  action='store_true', dest='forceclean', default=False,
+                  help='force revert of all Chromium changes, deletion of '+\
+                       'all unversioned files including the CEF folder and '+\
+                       'trigger the force-update, force-build and '+\
+                       'force-distrib options')
+parser.add_option('--force-update',
+                  action='store_true', dest='forceupdate', default=False,
+                  help='force Chromium and CEF update')
+parser.add_option('--force-build',
+                  action='store_true', dest='forcebuild', default=False,
+                  help='force CEF debug and release builds')
+parser.add_option('--force-distrib',
+                  action='store_true', dest='forcedistrib', default=False,
+                  help='force creation of CEF binary distribution')
+parser.add_option('--no-debug-build',
+                  action='store_true', dest='nodebugbuild', default=False,
+                  help="don't perform the CEF debug build")
+parser.add_option('--no-release-build',
+                  action='store_true', dest='noreleasebuild', default=False,
+                  help="don't perform the CEF release build")
+parser.add_option('--no-distrib',
+                  action='store_true', dest='nodistrib', default=False,
+                  help="don't create the CEF binary distribution")
+(options, args) = parser.parse_args()
+
+# the downloaddir option is required
+if options.downloaddir is None:
+  parser.print_help(sys.stderr)
+  sys.exit()
+
+# script directory
+script_dir = os.path.dirname(__file__)
+
+if not options.url is None:
+  # set the CEF URL
+  cef_url = options.url
+
+if not options.revision is None:
+  # set the CEF revision
+  cef_rev = str(options.revision)
+else:
+  # retrieve the CEF revision from the remote repo
+  info = get_svn_info(cef_url)
+  cef_rev = info['revision']
+
+# Retrieve the Chromium URL and revision from the CEF repo
+compat_url = cef_url + "/CHROMIUM_BUILD_COMPATIBILITY.txt?r="+cef_rev
+
+try:
+  # Read the remote URL contents
+  handle = urllib.urlopen(compat_url)
+  compat_value = handle.read().strip()
+  handle.close()
+
+  # Parse the contents
+  config = eval(compat_value, {'__builtins__': None}, None)
+
+  if not 'chromium_url' in config:
+    raise Exception("Missing chromium_url value")
+  if not 'chromium_revision' in config:
+    raise Exception("Missing chromium_revision value")
+  
+  chromium_url = config['chromium_url']
+  chromium_rev = config['chromium_revision']
+except Exception, e:
+  sys.stderr.write('Failed to read URL and revision information from '+ \
+                   compat_url+"\n")
+  raise
+
+download_dir = options.downloaddir
+if not os.path.exists(download_dir):
+  # create the download directory
+  os.makedirs(download_dir)
+
+# check the operating system
+platform = '';
+script_ext = '';
+if sys.platform == 'win32':
+  platform = 'windows'
+  script_ext = '.bat'
+elif sys.platform == 'darwin':
+  platform = 'macosx'
+  script_ext = '.sh'
+elif sys.platform == 'linux2':
+  platform = 'linux'
+  script_ext = '.sh'
+
+# check if the "depot_tools" directory exists
+depot_tools_dir = os.path.join(download_dir, 'depot_tools')
+if not os.path.exists(depot_tools_dir):
+  # checkout depot_tools
+  run('svn checkout '+depot_tools_url+' '+depot_tools_dir, download_dir);
+
+# check if the "chromium" directory exists
+chromium_dir = os.path.join(download_dir, 'chromium')
+if not os.path.exists(chromium_dir):
+  # create the "chromium" directory
+  os.makedirs(chromium_dir)
+
+chromium_src_dir = os.path.join(chromium_dir, 'src')
+cef_src_dir = os.path.join(chromium_src_dir, 'cef')
+cef_tools_dir = os.path.join(cef_src_dir, 'tools')
+
+# retrieve the current CEF URL and revision
+info = get_svn_info(cef_src_dir)
+current_cef_url = info['url']
+current_cef_rev = info['revision']
+
+# retrieve the current Chromium URL and revision
+info = get_svn_info(chromium_src_dir)
+current_chromium_url = info['url']
+current_chromium_rev = info['revision']
+
+# test if the CEF URL changed
+cef_url_changed = current_cef_url != cef_url
+sys.stdout.write('CEF URL: '+current_cef_url+"\n")
+if cef_url_changed:
+  sys.stdout.write('  -> CHANGED TO: '+cef_url+"\n")
+
+# test if the CEF revision changed
+cef_rev_changed = current_cef_rev != cef_rev
+sys.stdout.write('CEF Revision: '+current_cef_rev+"\n")
+if cef_rev_changed:
+  sys.stdout.write('  -> CHANGED TO: '+cef_rev+"\n")
+
+# test if the Chromium URL changed
+chromium_url_changed = current_chromium_url != chromium_url
+sys.stdout.write('Chromium URL: '+current_chromium_url+"\n")
+if chromium_url_changed:
+  sys.stdout.write('  -> CHANGED TO: '+chromium_url+"\n")
+
+# test if the Chromium revision changed
+chromium_rev_changed = current_chromium_rev != chromium_rev
+sys.stdout.write('Chromium Revision: '+current_chromium_rev+"\n")
+if chromium_rev_changed:
+  sys.stdout.write('  -> CHANGED TO: '+chromium_rev+"\n")
+
+# true if anything changed
+any_changed = chromium_url_changed or chromium_rev_changed or \
+              cef_url_changed or cef_rev_changed
+if not any_changed:
+  sys.stdout.write("No changes.\n")
+              
+if chromium_url_changed or options.forceconfig:
+  # run gclient config to create the .gclient file
+  run('gclient config '+chromium_url, chromium_dir, depot_tools_dir)
+  
+  path = os.path.join(chromium_dir, '.gclient')
+  if not os.path.exists(path):
+    sys.stderr.write(".gclient file was not created\n")
+    raise Exception('.gclient file was not created')
+  
+  # read the resulting .gclient file
+  fp = open(path, 'r')
+  data = fp.read()
+  fp.close()
+  
+  # populate "custom_deps" section
+  data = data.replace('"custom_deps" : {', '"custom_deps" : {'+\
+      "\n      "+'"src/third_party/WebKit/LayoutTests": None,'+\
+      "\n      "+'"src/chrome_frame/tools/test/reference_build/chrome": None,'+\
+      "\n      "+'"src/chrome/tools/test/reference_build/chrome_mac": None,'+\
+      "\n      "+'"src/chrome/tools/test/reference_build/chrome_win": None,'+\
+      "\n      "+'"src/chrome/tools/test/reference_build/chrome_linux": None,')
+  
+  # write the new .gclient file
+  fp = open(path, 'w')
+  fp.write(data)
+  fp.close()
+
+if options.forceclean:
+  if os.path.exists(chromium_src_dir):
+    # revert all Chromium changes and delete all unversioned files
+    run('gclient revert -n', chromium_dir, depot_tools_dir)
+  
+  # force update, build and distrib steps
+  options.forceupdate = True
+  options.forcebuild = True
+  options.forcedistrib = True
+  
+if chromium_url_changed or chromium_rev_changed or options.forceupdate:
+  # download/update the Chromium source code
+  run('gclient sync --revision src@'+chromium_rev+' --jobs 8 --force', \
+      chromium_dir, depot_tools_dir)
+
+if not os.path.exists(cef_src_dir) or cef_url_changed:
+  if cef_url_changed and os.path.exists(cef_src_dir):
+    # delete the cef directory (it will be re-downloaded)
+    shutil.rmtree(cef_src_dir)
+    
+  # download the CEF source code
+  run('svn checkout '+cef_url+' -r '+cef_rev+' '+cef_src_dir, download_dir)
+elif cef_rev_changed or options.forceupdate:
+  # update the CEF source code
+  run('svn update -r '+cef_rev+' '+cef_src_dir, download_dir)
+
+if any_changed or options.forceupdate:
+  # create CEF projects
+  run('cef_create_projects'+script_ext, cef_src_dir, depot_tools_dir)
+
+if any_changed or options.forcebuild:
+  if not options.nodebugbuild:
+    # make CEF Debug build
+    run('build_projects'+script_ext+' Debug', cef_tools_dir, depot_tools_dir)
+  
+  if not options.noreleasebuild:
+    # make CEF Release build
+    run('build_projects'+script_ext+' Release', cef_tools_dir, depot_tools_dir)
+  
+if any_changed or options.forcedistrib:
+  if not options.nodistrib:
+    # make CEF binary distribution
+    run('make_distrib'+script_ext, cef_tools_dir, depot_tools_dir)