Add ability to patch Chromium DEPS file (issue #1612)

tools/automate/automate-git.py

@@ -216,7 +216,7 @@ def write_branch_config_file(path, branch):
     write_config_file(config_file, {'branch': branch})
 
 def remove_deps_entry(path, entry):
-  """ Remove an entry from the DEPS file at the specified path. """
+  """ Remove an entry from the Chromium DEPS file at the specified path. """
   msg('Updating DEPS file: %s' % path)
   if not options.dryrun:
     if not os.path.isfile(path):
@@ -241,6 +241,22 @@ def remove_deps_entry(path, entry):
       fp.write(line)
     fp.close()
 
+def apply_deps_patch():
+  """ Patch the Chromium DEPS file if necessary. """
+  deps_file = '.DEPS.git'
+  patch_file = os.path.join(cef_dir, 'patch', 'patches', deps_file + '.patch')
+  if os.path.exists(patch_file):
+    # Attempt to apply the DEPS patch file that may exist with newer branches.
+    patch_tool = os.path.join(cef_dir, 'tools', 'patcher.py')
+    run('%s %s --patch-file "%s" --patch-dir "%s"' %
+            (python_exe, patch_tool, patch_file, chromium_src_dir),
+        chromium_src_dir, depot_tools_dir)
+  elif cef_branch != 'trunk':
+    # Older release branch DEPS files may include a 'src' entry. This entry
+    # needs to be removed otherwise `gclient sync` will fail.
+    deps_path = os.path.join(chromium_src_dir, deps_file)
+    remove_deps_entry(deps_path, "'src'")
+
 def onerror(func, path, exc_info):
   """
   Error handler for ``shutil.rmtree``.
@@ -526,17 +542,20 @@ if not options.noupdate:
   else:
     run('update_depot_tools', depot_tools_dir, depot_tools_dir);
 
-# Determine the git executables to use.
+# Determine the executables to use.
 if platform == 'windows':
   # Force use of the version bundled with depot_tools.
   git_exe = os.path.join(depot_tools_dir, 'git.bat')
+  python_exe = os.path.join(depot_tools_dir, 'python.bat')
   if options.dryrun and not os.path.exists(git_exe):
     sys.stdout.write("WARNING: --dry-run assumes that depot_tools" \
                      " is already in your PATH. If it isn't\nplease" \
                      " specify a --depot-tools-dir value.\n")
     git_exe = 'git.bat'
+    python_exe = 'python.bat'
 else:
   git_exe = 'git'
+  python_exe = 'git'
 
 
 ##
@@ -652,6 +671,7 @@ if not os.path.exists(gclient_file) or options.forceconfig:
           "u'build/scripts/command_wrapper/bin': None, "+\
           "u'build/scripts/gsd_generate_index': None, "+\
           "u'build/scripts/private/data/reliability': None, "+\
+          "u'build/scripts/tools/deps2git': None, "+\
           "u'build/third_party/lighttpd': None, "+\
           "u'commit-queue': None, "+\
           "u'depot_tools': None, "+\
@@ -686,7 +706,6 @@ if os.path.exists(chromium_src_dir):
   msg("Chromium URL: %s" % (get_git_url(chromium_src_dir)))
 
 # Determine the Chromium checkout options required by CEF.
-chromium_nohooks = False
 if options.chromiumcheckout == '':
   # Read the build compatibility file to identify the checkout name.
   compat_path = os.path.join(cef_dir, 'CHROMIUM_BUILD_COMPATIBILITY.txt')
@@ -696,10 +715,6 @@ if options.chromiumcheckout == '':
     chromium_checkout = config['chromium_checkout']
   else:
     raise Exception("Missing chromium_checkout value in %s" % (compat_path))
-
-  # Some branches run hooks using CEF instead of Chromium.
-  if 'chromium_nohooks' in config:
-    chromium_nohooks = config['chromium_nohooks']
 else:
   chromium_checkout = options.chromiumcheckout
 
@@ -755,20 +770,16 @@ if chromium_checkout_changed:
     (git_exe, ('--force ' if options.forceclean else ''), chromium_checkout), \
     chromium_src_dir, depot_tools_dir)
 
-  if cef_branch != 'trunk':
+  # Patch the Chromium DEPS file if necessary.
-    # Remove the 'src' entry from .DEPS.git for release branches.
+  apply_deps_patch()
-    # Otherwise, `gclient sync` will fail.
-    deps_path = os.path.join(chromium_src_dir, '.DEPS.git')
-    remove_deps_entry(deps_path, "'src'")
 
   # Set the GYP_CHROMIUM_NO_ACTION value temporarily so that `gclient sync` does
   # not run gyp.
   os.environ['GYP_CHROMIUM_NO_ACTION'] = '1'
 
   # Update third-party dependencies including branch/tag information.
-  run("gclient sync %s%s--with_branch_heads --jobs 16" % \
+  run("gclient sync %s--with_branch_heads --jobs 16" % \
-      (('--reset ' if options.forceclean else ''), \
+      (('--reset ' if options.forceclean else '')), \
-      ('--nohooks ' if chromium_nohooks else '')), \
       chromium_dir, depot_tools_dir)
 
   # Clear the GYP_CHROMIUM_NO_ACTION value.

tools/patch_util.py

@@ -9,6 +9,9 @@
     CEF Changes
     -----------
 
+    2015/04/22
+        - Write to stdout instead of using warning() for messages
+
     2013/01/03
         - Add support for patches containing new files
 
@@ -28,7 +31,7 @@ import re
 from stat import *
 # cStringIO doesn't support unicode in 2.5
 from StringIO import StringIO
-from logging import debug, info, warning
+from logging import debug, info
 
 from os.path import exists, isfile
 from os import unlink
@@ -55,6 +58,11 @@ def from_string(s):
          )
 
 
+def msg(message):
+  """ Output a message. """
+  sys.stdout.write('--> ' + message + "\n")
+
+
 class HunkInfo(object):
   """ parsed hunk data (hunk starts with @@ -R +R @@) """
 
@@ -160,7 +168,7 @@ class PatchInfo(object):
             hunkinfo.text.append(line)
             # todo: handle \ No newline cases
         else:
-            warning("invalid hunk no.%d at %d for target file %s" % (nexthunkno, lineno+1, self.target[nextfileno-1]))
+            msg("invalid hunk no.%d at %d for target file %s" % (nexthunkno, lineno+1, self.target[nextfileno-1]))
             # add hunk status node
             self.hunks[nextfileno-1].append(hunkinfo.copy())
             self.hunks[nextfileno-1][nexthunkno-1]["invalid"] = True
@@ -170,7 +178,7 @@ class PatchInfo(object):
 
         # check exit conditions
         if hunkactual["linessrc"] > hunkinfo.linessrc or hunkactual["linestgt"] > hunkinfo.linestgt:
-            warning("extra hunk no.%d lines at %d for target %s" % (nexthunkno, lineno+1, self.target[nextfileno-1]))
+            msg("extra hunk no.%d lines at %d for target %s" % (nexthunkno, lineno+1, self.target[nextfileno-1]))
             # add hunk status node
             self.hunks[nextfileno-1].append(hunkinfo.copy())
             self.hunks[nextfileno-1][nexthunkno-1]["invalid"] = True
@@ -186,7 +194,7 @@ class PatchInfo(object):
             # detect mixed window/unix line ends
             ends = self.hunkends[nextfileno-1]
             if ((ends["cr"]!=0) + (ends["crlf"]!=0) + (ends["lf"]!=0)) > 1:
-              warning("inconsistent line ends in patch hunks for %s" % self.source[nextfileno-1])
+              msg("inconsistent line ends in patch hunks for %s" % self.source[nextfileno-1])
             if debugmode:
               debuglines = dict(ends)
               debuglines.update(file=self.target[nextfileno-1], hunk=nexthunkno)
@@ -208,14 +216,14 @@ class PatchInfo(object):
       if filenames:
         if line.startswith("--- "):
           if nextfileno in self.source:
-            warning("skipping invalid patch for %s" % self.source[nextfileno])
+            msg("skipping invalid patch for %s" % self.source[nextfileno])
             del self.source[nextfileno]
             # double source filename line is encountered
             # attempt to restart from this second line
           re_filename = "^--- ([^\t]+)"
           match = re.match(re_filename, line)
           if not match:
-            warning("skipping invalid filename at line %d" % lineno)
+            msg("skipping invalid filename at line %d" % lineno)
             # switch back to header state
             filenames = False
             header = True
@@ -223,16 +231,16 @@ class PatchInfo(object):
             self.source.append(match.group(1).strip())
         elif not line.startswith("+++ "):
           if nextfileno in self.source:
-            warning("skipping invalid patch with no target for %s" % self.source[nextfileno])
+            msg("skipping invalid patch with no target for %s" % self.source[nextfileno])
             del self.source[nextfileno]
           else:
             # this should be unreachable
-            warning("skipping invalid target patch")
+            msg("skipping invalid target patch")
           filenames = False
           header = True
         else:
           if nextfileno in self.target:
-            warning("skipping invalid patch - double target at line %d" % lineno)
+            msg("skipping invalid patch - double target at line %d" % lineno)
             del self.source[nextfileno]
             del self.target[nextfileno]
             nextfileno -= 1
@@ -244,7 +252,7 @@ class PatchInfo(object):
             re_filename = "^\+\+\+ ([^\t]+)"
             match = re.match(re_filename, line)
             if not match:
-              warning("skipping invalid patch - no target filename at line %d" % lineno)
+              msg("skipping invalid patch - no target filename at line %d" % lineno)
               # switch back to header state
               filenames = False
               header = True
@@ -264,7 +272,7 @@ class PatchInfo(object):
         match = re.match("^@@ -(\d+)(,(\d+))? \+(\d+)(,(\d+))?", line)
         if not match:
           if nextfileno-1 not in self.hunks:
-            warning("skipping invalid patch with no hunks for file %s" % self.target[nextfileno-1])
+            msg("skipping invalid patch with no hunks for file %s" % self.target[nextfileno-1])
             # switch to header state
             hunkhead = False
             header = True
@@ -296,7 +304,7 @@ class PatchInfo(object):
           continue
     else:
       if not hunkskip:
-        warning("patch file incomplete - %s" % filename)
+        msg("patch file incomplete - %s" % filename)
         # sys.exit(?)
       else:
         # duplicated message when an eof is reached
@@ -319,7 +327,7 @@ class PatchInfo(object):
         if len(self.hunks[fileno]) == 1 and self.hunks[fileno][0].startsrc == 0:
           hunklines = [x[1:].rstrip("\r\n") for x in self.hunks[fileno][0].text if x[0] in " +"]
           if len(hunklines) > 0:
-            warning("creating file %s" % (f2patch))
+            msg("creating file %s" % (f2patch))
             f = open(f2patch, "wb")
             for line in hunklines:
               f.write(line + "\n")
@@ -328,10 +336,10 @@ class PatchInfo(object):
 
         f2patch = self.target[fileno]
         if not exists(f2patch):
-          warning("source/target file does not exist\n--- %s\n+++ %s" % (filename, f2patch))
+          msg("source/target file does not exist\n--- %s\n+++ %s" % (filename, f2patch))
           continue
       if not isfile(f2patch):
-        warning("not a file - %s" % f2patch)
+        msg("not a file - %s" % f2patch)
         continue
       filename = f2patch
 
@@ -385,29 +393,29 @@ class PatchInfo(object):
       else:
         if hunkno < len(self.hunks[fileno]) and \
             (len(self.hunks[fileno]) != 1 or self.hunks[fileno][0].startsrc != 0):
-          warning("premature end of source file %s at hunk %d" % (filename, hunkno+1))
+          msg("premature end of source file %s at hunk %d" % (filename, hunkno+1))
 
       f2fp.close()
 
       if validhunks < len(self.hunks[fileno]):
         if check_patched(filename, self.hunks[fileno]):
-          warning("already patched  %s" % filename)
+          msg("already patched  %s" % filename)
         else:
-          warning("source file is different - %s" % filename)
+          msg("source file is different - %s" % filename)
       if canpatch:
         backupname = filename+".orig"
         if exists(backupname):
-          warning("can't backup original file to %s - aborting" % backupname)
+          msg("can't backup original file to %s - aborting" % backupname)
         else:
           import shutil
           shutil.move(filename, backupname)
           if patch_hunks(backupname, filename, self.hunks[fileno]):
-            warning("successfully patched %s" % filename)
+            msg("successfully patched %s" % filename)
             unlink(backupname)
           else:
-            warning("error patching file %s" % filename)
+            msg("error patching file %s" % filename)
             shutil.copy(filename, filename+".invalid")
-            warning("invalid version is saved to %s" % filename+".invalid")
+            msg("invalid version is saved to %s" % filename+".invalid")
             # todo: proper rejects
             shutil.move(backupname, filename)
 
@@ -461,7 +469,7 @@ def check_patched(filename, hunks):
           if not len(line):
             raise NoMatch
           if line.rstrip("\r\n") != hline[1:].rstrip("\r\n"):
-            warning("file is not patched - failed hunk: %d" % (hno+1))
+            msg("file is not patched - failed hunk: %d" % (hno+1))
             raise NoMatch
   except NoMatch:
     matched = False

tools/patcher.py

@@ -10,13 +10,61 @@ from file_util import *
 from patch_util import *
 
 
-# cannot be loaded as a module
+# Cannot be loaded as a module.
 if __name__ != "__main__":
-    sys.stderr.write('This file cannot be loaded as a module!')
+  sys.stdout.write('This file cannot be loaded as a module!')
-    sys.exit()
+  sys.exit()
 
 
-# parse command-line options
+def normalize_dir(dir):
+  ''' Normalize the directory value. '''
+  dir = dir.replace('\\', '/')
+  if dir[-1] != '/':
+    dir += '/'
+  return dir
+
+def patch_file(patch_file, patch_dir):
+  ''' Apply a single patch file in a single directory. '''
+  if not os.path.isfile(patch_file):
+    raise Exception('Patch file %s does not exist.' % patch_file)
+
+  sys.stdout.write('Reading patch file %s\n' % patch_file)
+  patchObj = from_file(patch_file)
+  patchObj.apply(normalize_dir(patch_dir))
+
+def patch_config(config_file):
+  ''' Apply patch files based on a configuration file. '''
+  # Normalize the patch directory value.
+  patchdir = normalize_dir(os.path.dirname(os.path.abspath(config_file)))
+
+  if not os.path.isfile(config_file):
+    raise Exception('Patch config file %s does not exist.' % config_file)
+
+  # Parse the configuration file.
+  scope = {}
+  execfile(config_file, scope)
+  patches = scope["patches"]
+
+  for patch in patches:
+    file = patchdir+'patches/'+patch['name']+'.patch'
+    dopatch = True
+
+    if 'condition' in patch:
+      # Check that the environment variable is set.
+      if patch['condition'] not in os.environ:
+        sys.stdout.write('Skipping patch file %s\n' % file)
+        dopatch = False
+
+    if dopatch:
+      patch_file(file, patch['path'])
+      if 'note' in patch:
+        separator = '-' * 79 + '\n'
+        sys.stdout.write(separator)
+        sys.stdout.write('NOTE: %s\n' % patch['note'])
+        sys.stdout.write(separator)
+
+
+# Parse command-line options.
 disc = """
 This utility applies patch files.
 """
@@ -24,80 +72,16 @@ This utility applies patch files.
 parser = OptionParser(description=disc)
 parser.add_option('--patch-config', dest='patchconfig', metavar='DIR',
                   help='patch configuration file')
+parser.add_option('--patch-file', dest='patchfile', metavar='FILE',
+                  help='patch source file')
+parser.add_option('--patch-dir', dest='patchdir', metavar='DIR',
+                  help='patch target directory')
 (options, args) = parser.parse_args()
 
-# the patchconfig option is required
+if not options.patchconfig is None:
-if options.patchconfig is None:
+  patch_config(options.patchconfig)
-    parser.print_help(sys.stdout)
+elif not options.patchfile is None and not options.patchdir is None:
-    sys.exit()
+  patch_file(options.patchfile, options.patchdir)
-
-# normalize the patch directory value
-patchdir = os.path.dirname(os.path.abspath(options.patchconfig)).replace('\\', '/')
-if patchdir[-1] != '/':
-    patchdir += '/'
-
-# check if the patching should be skipped
-if os.path.isfile(patchdir + 'NOPATCH'):
-    nopatch = True
-    sys.stdout.write('NOPATCH exists -- files have not been patched.\n')
 else:
-    nopatch = False
+  parser.print_help(sys.stdout)
-    # locate the patch configuration file
+  sys.exit()
-    if not os.path.isfile(options.patchconfig):
-        sys.stderr.write('File '+options.patchconfig+' does not exist.\n')
-        sys.exit()
-    
-    scope = {}
-    execfile(options.patchconfig, scope)
-    patches = scope["patches"]
-    
-    for patch in patches:
-        file = patchdir+'patches/'+patch['name']+'.patch'
-        dopatch = True
-
-        if 'condition' in patch:
-          # Check that the environment variable is set.
-          if patch['condition'] not in os.environ:
-            sys.stderr.write('Skipping patch file '+file+'\n')
-            dopatch = False
-
-        if dopatch:
-          if not os.path.isfile(file):
-              sys.stderr.write('Patch file '+file+' does not exist.\n')
-          else:
-              sys.stderr.write('Reading patch file '+file+'\n')
-              dir = patch['path']
-              patchObj = from_file(file)
-              patchObj.apply(dir)
-              if 'note' in patch:
-                  separator = '-' * 79 + '\n'
-                  sys.stderr.write(separator)
-                  sys.stderr.write('NOTE: '+patch['note']+'\n')
-                  sys.stderr.write(separator)
-
-# read the current include file, if any
-incfile = patchdir + 'patch_state.h'
-if nopatch:
-    incnew = """// This file is generated by the patch tool and should not be edited manually.
-#ifndef _PATCH_STATE_H
-#define _PATCH_STATE_H
-// No patches have been applied to the Chromium/WebKit source base.
-#define CEF_PATCHES_APPLIED    0
-#endif // _PATCH_STATE_H
-"""
-else:
-    incnew = """// This file is generated by the patch tool and should not be edited manually.
-#ifndef _PATCH_STATE_H
-#define _PATCH_STATE_H
-// Patches have been applied to the Chromium/WebKit source base.
-#define CEF_PATCHES_APPLIED    1
-#endif // _PATCH_STATE_H
-"""
-
-inccur = ''
-if os.path.isfile(incfile):
-    inccur = read_file(incfile)
-    
-if inccur != incnew:
-    sys.stdout.write('Writing file '+incfile+'.\n')
-    write_file(incfile, incnew)