Support faster incremental Chromium/CEF updates with automate-git.py --fast-update (issue #2435).

tools/automate/automate-git.py

@@ -284,6 +284,12 @@ def apply_deps_patch():
     raise Exception("Path does not exist: %s" % (deps_path))
 
 
+def run_patch_updater(args):
+  """ Run the patch updater script. """
+  tool = os.path.join(cef_src_dir, 'tools', 'patch_updater.py')
+  run('%s %s %s' % (python_exe, tool, args), cef_src_dir, depot_tools_dir)
+
+
 def onerror(func, path, exc_info):
   """
   Error handler for ``shutil.rmtree``.
@@ -413,6 +419,12 @@ parser.add_option(
     dest='nodepottoolsupdate',
     default=False,
     help='Do not update depot_tools.')
+parser.add_option('--fast-update',
+                  action='store_true', dest='fastupdate', default=False,
+                  help='Update existing Chromium/CEF checkouts for fast incremental '+\
+                       'builds by attempting to minimize the number of modified files. '+\
+                       'The update will fail if there are unstaged CEF changes or if '+\
+                       'Chromium changes are not included in a patch file.')
 
 # Build-related options.
 parser.add_option('--force-build',
@@ -561,7 +573,8 @@ if options.noupdate:
 if (options.nochromiumupdate and options.forceupdate) or \
    (options.nocefupdate and options.forceupdate) or \
    (options.nobuild and options.forcebuild) or \
-   (options.nodistrib and options.forcedistrib):
+   (options.nodistrib and options.forcedistrib) or \
+   ((options.forceclean or options.forcecleandeps) and options.fastupdate):
   print "Invalid combination of options."
   parser.print_help(sys.stderr)
   sys.exit()
@@ -696,12 +709,22 @@ if platform == 'windows':
   # Avoid errors when the "vs_toolchain.py update" Chromium hook runs.
   os.environ['DEPOT_TOOLS_WIN_TOOLCHAIN'] = '0'
 
+download_dir = os.path.abspath(options.downloaddir)
+chromium_dir = os.path.join(download_dir, 'chromium')
+chromium_src_dir = os.path.join(chromium_dir, 'src')
+out_src_dir = os.path.join(chromium_src_dir, 'out')
+cef_src_dir = os.path.join(chromium_src_dir, 'cef')
+
+if options.fastupdate and os.path.exists(cef_src_dir):
+  cef_dir = cef_src_dir
+else:
+  cef_dir = os.path.join(download_dir, 'cef')
+
 ##
 # Manage the download directory.
 ##
 
 # Create the download directory if necessary.
-download_dir = os.path.abspath(options.downloaddir)
 create_directory(download_dir)
 
 msg("Download Directory: %s" % (download_dir))
@@ -762,8 +785,6 @@ else:
 # Manage the cef directory.
 ##
 
-cef_dir = os.path.join(download_dir, 'cef')
-
 # Delete the existing CEF directory if requested.
 if options.forceclean and os.path.exists(cef_dir):
   delete_directory(cef_dir)
@@ -803,7 +824,7 @@ else:
 # Create the CEF checkout if necessary.
 if not options.nocefupdate and not os.path.exists(cef_dir):
   cef_checkout_new = True
-  run('%s clone %s %s' % (git_exe, cef_url, cef_dir), download_dir, \
+  run('%s clone %s %s' % (git_exe, cef_url, cef_dir), download_dir,
       depot_tools_dir)
 else:
   cef_checkout_new = False
@@ -848,13 +869,8 @@ msg("CEF Output Directory: %s" % (out_dir))
 ##
 
 # Create the chromium directory if necessary.
-chromium_dir = os.path.join(download_dir, 'chromium')
 create_directory(chromium_dir)
 
-chromium_src_dir = os.path.join(chromium_dir, 'src')
-cef_src_dir = os.path.join(chromium_src_dir, 'cef')
-out_src_dir = os.path.join(chromium_src_dir, 'out')
-
 if options.chromiumurl != '':
   chromium_url = options.chromiumurl
 else:
@@ -936,7 +952,8 @@ else:
 
 # Delete the existing src/cef directory. It will be re-copied from the download
 # directory later.
-if cef_checkout_changed and os.path.exists(cef_src_dir):
+if cef_checkout_changed and cef_dir != cef_src_dir and os.path.exists(
+    cef_src_dir):
   delete_directory(cef_src_dir)
 
 # Delete the existing src/out directory if requested.
@@ -955,7 +972,10 @@ if os.path.exists(out_src_dir):
 # Update the Chromium checkout.
 if chromium_checkout_changed:
   if not chromium_checkout_new:
-    if options.forceclean and options.forcecleandeps:
+    if options.fastupdate:
+      # Backup and revert the patched files.
+      run_patch_updater("--backup --revert")
+    elif options.forceclean and options.forcecleandeps:
       # Remove all local changes including third-party git checkouts managed by
       # gclient.
       run("%s clean -dffx" % (git_exe), chromium_src_dir, depot_tools_dir)
@@ -988,11 +1008,16 @@ if chromium_checkout_changed:
   # Clear the GYP_CHROMIUM_NO_ACTION value.
   del os.environ['GYP_CHROMIUM_NO_ACTION']
 
+  if not chromium_checkout_new and options.fastupdate:
+    # Check and restore the patched files.
+    run_patch_updater("--reapply --restore")
+
   # Delete the src/out directory created by `gclient sync`.
   delete_directory(out_src_dir)
 
 # Restore the src/cef directory.
-if os.path.exists(cef_dir) and not os.path.exists(cef_src_dir):
+if cef_dir != cef_src_dir and os.path.exists(
+    cef_dir) and not os.path.exists(cef_src_dir):
   copy_directory(cef_dir, cef_src_dir)
 
 # Restore the src/out directory.

tools/file_util.py

@@ -52,7 +52,7 @@ def backup_file(name):
 def copy_file(src, dst, quiet=True):
   """ Copy a file. """
   try:
-    shutil.copy(src, dst)
+    shutil.copy2(src, dst)
     if not quiet:
       sys.stdout.write('Transferring ' + src + ' file.\n')
   except IOError, (errno, strerror):

tools/patch_updater.py

@@ -7,8 +7,11 @@ import os
 import re
 import sys
 from exec_util import exec_cmd
+from file_util import copy_file, move_file, read_file, remove_file
 import git_util as git
 
+backup_ext = '.cefbak'
+
 
 def msg(message):
   """ Output a message. """
@@ -67,7 +70,7 @@ parser.add_option(
     action='store_true',
     dest='resave',
     default=False,
-    help='re-save existing patch files to pick up manual changes')
+    help='resave existing patch files to pick up manual changes')
 parser.add_option(
     '--reapply',
     action='store_true',
@@ -80,6 +83,19 @@ parser.add_option(
     dest='revert',
     default=False,
     help='revert all changes from existing patch files')
+parser.add_option(
+    '--backup',
+    action='store_true',
+    dest='backup',
+    default=False,
+    help='backup patched files. Used in combination with --revert.')
+parser.add_option(
+    '--restore',
+    action='store_true',
+    dest='restore',
+    default=False,
+    help='restore backup of patched files that have not changed. If a backup has ' +\
+         'changed the patch file will be resaved. Used in combination with --reapply.')
 parser.add_option(
     '--patch',
     action='extend',
@@ -136,6 +152,9 @@ for patch in patches:
     # List of paths added by the patch file.
     added_paths = []
 
+    # True if any backed up files have changed.
+    has_backup_changes = False
+
     if not options.resave:
       if not options.reapply:
         # Revert any changes to existing files in the patch.
@@ -143,6 +162,14 @@ for patch in patches:
           patch_path_abs = os.path.abspath(os.path.join(patch_root_abs, \
                                                         patch_path))
           if os.path.exists(patch_path_abs):
+            if options.backup:
+              backup_path_abs = patch_path_abs + backup_ext
+              if not os.path.exists(backup_path_abs):
+                msg('Creating backup of %s' % patch_path_abs)
+                copy_file(patch_path_abs, backup_path_abs)
+              else:
+                msg('Skipping backup of %s' % patch_path_abs)
+
             msg('Reverting changes to %s' % patch_path_abs)
             cmd = 'git checkout -- %s' % (patch_path_abs)
             result = exec_cmd(cmd, patch_root_abs)
@@ -170,7 +197,25 @@ for patch in patches:
                patch['name'])
           continue
 
-    if not options.revert and not options.reapply:
+        if options.restore:
+          # Restore from backup if a backup exists.
+          for patch_path in patch_paths:
+            patch_path_abs = os.path.abspath(os.path.join(patch_root_abs, \
+                                                          patch_path))
+            backup_path_abs = patch_path_abs + backup_ext
+            if os.path.exists(backup_path_abs):
+              if read_file(patch_path_abs) == read_file(backup_path_abs):
+                msg('Restoring backup of %s' % patch_path_abs)
+                remove_file(patch_path_abs)
+                move_file(backup_path_abs, patch_path_abs)
+              else:
+                msg('Discarding backup of %s' % patch_path_abs)
+                remove_file(backup_path_abs)
+                has_backup_changes = True
+            else:
+              msg('No backup of %s' % patch_path_abs)
+
+    if (not options.revert and not options.reapply) or has_backup_changes:
       msg('Saving changes to %s' % patch_file)
       if added_paths:
         # Inform git of the added paths so they appear in the patch file.