Support analysis of Chromium update status with automate-git.py --log-chromium-changes (issue #2435)

CHROMIUM_UPDATE.txt

@@ -0,0 +1,64 @@
+# The Chromium Embedded Framework (CEF) project is built on top of the Chromium
+# project source tree. When updating Chromium to a new version certain files and
+# patterns should be observed for changes. If changes are detected then the CEF
+# source code or patch files will likely need to be updated.
+#
+# Add `--log-chromium-changes` to the automate-git.py command-line to output
+# the following files in the <download-dir>:
+#
+# * chromium_update_changes.diff
+#   Files in the chromium/src directory that have changed. See the 'files'
+#   section below.
+#
+# * chromium_update_patterns.txt
+#   Files in the chromium/src directory that contain invalid/unexpected
+#   patterns. See the 'patterns' section below. Failure of this step is
+#   considered a fatal error during update.
+#
+# * chromium_update_patches.txt
+#   Output from attempting to update existing Chromium patch files using the
+#   patch_updater.py tool. Failure of this step is considered a fatal error
+#   during update.
+#
+# For complete update instructions see:
+# https://bitbucket.org/chromiumembedded/cef/wiki/ChromiumUpdate.md
+
+{
+  # Files in the chromium/src directory that should be evaluated for changes.
+  # Similar changes may need to be applied to the CEF source code.
+  'files': [
+     'chrome/browser/extensions/api/tabs/tabs_api.*',
+     'chrome/browser/extensions/chrome_component_extension_resource_manager.*',
+     'chrome/browser/extensions/chrome_extension_web_contents_observer.*',
+     'chrome/browser/extensions/component_loader.*',
+     'chrome/browser/extensions/extension_service.*',
+     'chrome/browser/printing/print_view_manager*',
+     'chrome/browser/printing/printing_message_filter*',
+     'chrome/browser/renderer_host/chrome_resource_dispatcher_host_delegate.*',
+     'chrome/common/extensions/api/*_features.json',
+     'chrome/renderer/chrome_content_renderer_client.*',
+     'chrome/renderer/extensions/chrome_extensions_renderer_client.*',
+     'content/shell/BUILD.gn',
+     'content/shell/app/*',
+     'content/shell/browser/shell_*',
+     'content/shell/browser/renderer_host/shell_*',
+     'content/shell/common/shell_*',
+     'content/shell/gpu/shell_*',
+     'content/shell/renderer/shell_*',
+     'content/shell/utility/shell_*',
+     'extensions/shell/*',
+  ],
+  # Patterns that should not be found in the chromium/src directory after
+  # applying patch files.
+  'patterns': [
+    {
+      # New instances of this static_cast are added to the Chromium sources with
+      # some regularity. If unfixed they will result in runtime crashes.
+      'pattern': 'static_cast<StoragePartitionImpl\*>(',
+      'exclude_matches': '^(.+?)test(.+?):',
+      'message': 'New instances in non-test files should be converted to ' +\
+                 'call StoragePartition methods.' +\
+                 '\nSee storage_partition_1973.patch.',
+    },
+  ],
+}

tools/automate/automate-git.py

@@ -59,14 +59,19 @@ def run(command_line, working_dir, depot_tools_dir=None, output_file=None):
     if not output_file:
       return subprocess.check_call(
           args, cwd=working_dir, env=env, shell=(sys.platform == 'win32'))
-    with open(output_file, "w") as f:
+    try:
-      return subprocess.check_call(
+      msg('Writing %s' % output_file)
-          args,
+      with open(output_file, "w") as f:
-          cwd=working_dir,
+        return subprocess.check_call(
-          env=env,
+            args,
-          shell=(sys.platform == 'win32'),
+            cwd=working_dir,
-          stderr=subprocess.STDOUT,
+            env=env,
-          stdout=f)
+            shell=(sys.platform == 'win32'),
+            stderr=subprocess.STDOUT,
+            stdout=f)
+    except subprocess.CalledProcessError:
+      msg('ERROR Run failed. See %s for output.' % output_file)
+      raise
 
 
 def create_directory(path):
@@ -300,12 +305,13 @@ def apply_deps_patch():
     raise Exception("Path does not exist: %s" % (deps_path))
 
 
-def run_patch_updater(args=''):
+def run_patch_updater(args='', output_file=None):
   """ Run the patch updater script. """
   tool = os.path.join(cef_src_dir, 'tools', 'patch_updater.py')
   if len(args) > 0:
     args = ' ' + args
-  run('%s %s%s' % (python_exe, tool, args), cef_src_dir, depot_tools_dir)
+  run('%s %s%s' % (python_exe, tool, args), cef_src_dir, depot_tools_dir,
+      output_file)
 
 
 def onerror(func, path, exc_info):
@@ -499,6 +505,103 @@ def get_build_directory_name(is_debug):
   return build_dir
 
 
+def read_update_file():
+  update_path = os.path.join(cef_src_dir, 'CHROMIUM_UPDATE.txt')
+  if not os.path.exists(update_path):
+    msg("Missing file: %s" % update_path)
+    return None
+
+  msg("Reading %s" % update_path)
+  return read_config_file(update_path)
+
+
+def log_chromium_changes():
+  """ Evaluate the Chromium checkout for changes. """
+  config = read_update_file()
+  if config is None:
+    msg("Skipping Chromium changes log.")
+    return
+
+  if 'files' in config:
+    out_file = os.path.join(download_dir, 'chromium_update_changes.diff')
+    if os.path.exists(out_file):
+      os.remove(out_file)
+
+    old_commit = get_chromium_master_commit(
+        get_chromium_master_position(chromium_compat_version))
+    new_commit = get_chromium_master_commit(
+        get_chromium_master_position(chromium_checkout))
+
+    cmd = '%s diff --relative --no-prefix %s..%s -- %s' % (
+        git_exe, old_commit, new_commit, ' '.join(config['files']))
+    result = exec_cmd(cmd, chromium_src_dir)
+    if result['out'] != '':
+      msg('Writing %s' % out_file)
+      with open(out_file, 'w') as fp:
+        fp.write(result['out'])
+
+
+def check_pattern_matches(output_file=None):
+  """ Evaluate the Chromium checkout for pattern matches. """
+  config = read_update_file()
+  if config is None:
+    msg("Skipping Chromium pattern matching.")
+    return
+
+  if 'patterns' in config:
+    if output_file is None:
+      fp = sys.stdout
+    else:
+      msg('Writing %s' % output_file)
+      fp = open(output_file, 'w')
+
+    has_output = False
+    for entry in config['patterns']:
+      msg("Evaluating pattern: %s" % entry['pattern'])
+
+      # Read patterns from a file to avoid formatting problems.
+      pattern_handle, pattern_file = tempfile.mkstemp()
+      os.write(pattern_handle, entry['pattern'])
+      os.close(pattern_handle)
+
+      cmd = '%s grep -n -f %s' % (git_exe, pattern_file)
+      result = exec_cmd(cmd, chromium_src_dir)
+      os.remove(pattern_file)
+
+      if result['out'] != '':
+        write_msg = True
+        re_exclude = re.compile(
+            entry['exclude_matches']) if 'exclude_matches' in entry else None
+
+        for line in result['out'].split('\n'):
+          line = line.strip()
+          if len(line) == 0:
+            continue
+          skip = not re_exclude is None and re_exclude.match(line) != None
+          if not skip:
+            if write_msg:
+              if has_output:
+                fp.write('\n')
+              fp.write('!!!! WARNING: FOUND PATTERN: %s\n' % entry['pattern'])
+              if 'message' in entry:
+                fp.write(entry['message'] + '\n')
+              fp.write('\n')
+              write_msg = False
+            fp.write(line + '\n')
+            has_output = True
+
+    if not output_file is None:
+      if has_output:
+        msg('ERROR Matches found. See %s for output.' % out_file)
+      else:
+        fp.write('Good news! No matches.\n')
+      fp.close()
+
+    if has_output:
+      # Don't continue when we know the build will be wrong.
+      sys.exit(1)
+
+
 ##
 # Program entry point.
 ##
@@ -624,6 +727,18 @@ parser.add_option('--fast-update',
                        'builds by attempting to minimize the number of modified files. '+\
                        'The update will fail if there are unstaged CEF changes or if '+\
                        'Chromium changes are not included in a patch file.')
+parser.add_option(
+    '--force-patch-update',
+    action='store_true',
+    dest='forcepatchupdate',
+    default=False,
+    help='Force update of patch files.')
+parser.add_option(
+    '--log-chromium-changes',
+    action='store_true',
+    dest='logchromiumchanges',
+    default=False,
+    help='Create a log of the Chromium changes.')
 
 # Build-related options.
 parser.add_option('--force-build',
@@ -1282,10 +1397,29 @@ elif not out_src_dir_exists:
 # Write the config file for identifying the branch.
 write_branch_config_file(out_src_dir, cef_branch)
 
-if not options.fastupdate and chromium_checkout_changed and \
+if options.logchromiumchanges and chromium_checkout != chromium_compat_version:
-   chromium_checkout != chromium_compat_version:
+  log_chromium_changes()
+
+if options.forcepatchupdate or (not options.fastupdate and \
+                                chromium_checkout_changed and \
+                                chromium_checkout != chromium_compat_version):
   # Not using the known-compatible Chromium version. Try to update patch files.
-  run_patch_updater()
+  if options.logchromiumchanges:
+    out_file = os.path.join(download_dir, 'chromium_update_patches.txt')
+    if os.path.exists(out_file):
+      os.remove(out_file)
+  else:
+    out_file = None
+  run_patch_updater(output_file=out_file)
+
+if chromium_checkout != chromium_compat_version:
+  if options.logchromiumchanges:
+    out_file = os.path.join(download_dir, 'chromium_update_patterns.txt')
+    if os.path.exists(out_file):
+      os.remove(out_file)
+  else:
+    out_file = None
+  check_pattern_matches(output_file=out_file)
 
 ##
 # Build CEF.