From 236d419d9d3e71cca7f458a332d95376fd9c4ed0 Mon Sep 17 00:00:00 2001 From: Marshall Greenblatt Date: Sat, 14 Mar 2015 23:27:08 -0400 Subject: [PATCH] Improvements to automate-git.py - Remove support for SVN checkouts (issue #1577). - Use new Bitbucket cef.git repo for Git checkouts (issue #1577). - Windows: Set DEPOT_TOOLS_WIN_TOOLCHAIN=0 to avoid errors while running Chromium hooks (issue #1533). - Fix create/restore of out_ directories. --- tools/automate/automate-git.py | 193 +++++++------------------- tools/automate/gitsvnmirror.py | 241 --------------------------------- 2 files changed, 49 insertions(+), 385 deletions(-) delete mode 100644 tools/automate/gitsvnmirror.py diff --git a/tools/automate/automate-git.py b/tools/automate/automate-git.py index 84211fcac..1760a22e2 100644 --- a/tools/automate/automate-git.py +++ b/tools/automate/automate-git.py @@ -10,7 +10,6 @@ import subprocess import sys import tempfile import urllib -import xml.etree.ElementTree as ET import zipfile ## @@ -20,10 +19,7 @@ import zipfile depot_tools_url = 'https://chromium.googlesource.com/chromium/tools/depot_tools.git' depot_tools_archive_url = 'https://src.chromium.org/svn/trunk/tools/depot_tools.zip' -cef_git_trunk_url = 'https://chromiumembedded@bitbucket.org/chromiumembedded/trunk-cef3.git' -cef_git_branch_url = 'https://chromiumembedded@bitbucket.org/chromiumembedded/branches-%1-cef3.git' -cef_svn_trunk_url = 'https://chromiumembedded.googlecode.com/svn/trunk/cef3' -cef_svn_branch_url = 'https://chromiumembedded.googlecode.com/svn/branches/%1/cef3' +cef_git_url = 'https://chromiumembedded@bitbucket.org/chromiumembedded/cef.git' ## @@ -102,10 +98,6 @@ def is_git_checkout(path): """ Returns true if the path represents a git checkout. """ return os.path.exists(os.path.join(path, '.git')) -def is_svn_checkout(path): - """ Returns true if the path represents an svn checkout. """ - return os.path.exists(os.path.join(path, '.svn')) - def exec_cmd(cmd, path): """ Execute the specified command and return the result. """ out = '' @@ -140,36 +132,6 @@ def get_git_url(path): return result['out'].strip() return 'Unknown' -def get_git_svn_revision(path, branch): - """ Returns the SVN revision associated with the specified path and git - branch/tag/hash. """ - svn_rev = "None" - cmd = "%s log --grep=^git-svn-id: -n 1 %s" % (git_exe, branch) - result = exec_cmd(cmd, path) - if result['err'] == '': - for line in result['out'].split('\n'): - if line.find("git-svn-id") > 0: - svn_rev = line.split("@")[1].split()[0] - break - return svn_rev - -def get_svn_info(path): - """ Retrieves the URL and revision from svn info. """ - url = 'None' - rev = 'None' - cmd = "%s info --xml %s" % (svn_exe, path) - is_http = path[0:4] == 'http' - if is_http or os.path.exists(path): - result = exec_cmd(cmd, path if not is_http else '.') - if result['err'] == '': - tree = ET.ElementTree(ET.fromstring(result['out'])) - entry = tree.getroot().find('entry') - url = entry.find('url').text - rev = entry.attrib['revision'] - else: - raise Exception("Failed to execute svn info: %s" % (result['err'])) - return {'url': url, 'revision': rev} - def download_and_extract(src, target, contents_prefix): """ Extracts the contents of src, which may be a URL or local file, to the target directory. """ @@ -332,17 +294,12 @@ parser.add_option('--branch', dest='branch', default='trunk') parser.add_option('--url', dest='url', help='CEF download URL. If not specified the default URL '+\ - 'will be used for the chosen branch.', + 'will be used.', default='') parser.add_option('--checkout', dest='checkout', help='Version of CEF to checkout. If not specified the '+\ - 'most recent version of the branch will be used. '+\ - 'If --use-git is specified this should be a Git '+\ - 'branch/hash/tag instead of an SVN revision number.', + 'most recent remote version of the branch will be used.', default='') -parser.add_option('--use-svn', - action='store_true', dest='usesvn', default=False, - help="Download CEF source code using SVN instead of Git.") parser.add_option('--chromium-checkout', dest='chromiumcheckout', help='Version of Chromium to checkout (Git '+\ 'branch/hash/tag). This overrides the value specified '+\ @@ -496,6 +453,10 @@ if options.branch != 'trunk' and not options.branch.isdigit(): print 'Invalid branch value: %s' % (options.branch) cef_branch = options.branch +if cef_branch != 'trunk' and int(cef_branch) <= 1453: + print 'The requested branch is too old to build using this tool' + sys.exit() + # True if the requested branch is 2272 or newer. branch_is_2272_or_newer = (cef_branch == 'trunk' or int(cef_branch) >= 2272) @@ -512,6 +473,10 @@ gyp_needs_target_arch_x64 = options.x64build and \ # Options that force the sources to change. force_change = options.forceclean or options.forceupdate +if platform == 'windows': + # Avoid errors when the "vs_toolchain.py update" Chromium hook runs. + os.environ['DEPOT_TOOLS_WIN_TOOLCHAIN'] = '0' + ## # Manage the download directory. @@ -561,20 +526,16 @@ if not options.noupdate: else: run('update_depot_tools', depot_tools_dir, depot_tools_dir); -# Determine the svn/git executables to use. +# Determine the git executables to use. if platform == 'windows': # Force use of the version bundled with depot_tools. - svn_exe = os.path.join(depot_tools_dir, 'svn.bat') git_exe = os.path.join(depot_tools_dir, 'git.bat') - if options.dryrun and (not os.path.exists(svn_exe) or \ - not os.path.exists(git_exe)): + if options.dryrun and not os.path.exists(git_exe): sys.stdout.write("WARNING: --dry-run assumes that depot_tools" \ " is already in your PATH. If it isn't\nplease" \ " specify a --depot-tools-dir value.\n") - svn_exe = 'svn.bat' git_exe = 'git.bat' else: - svn_exe = 'svn' git_exe = 'git' @@ -582,49 +543,25 @@ else: # Manage the cef directory. ## -cef_dir = os.path.join(download_dir, 'cef_' + cef_branch) +cef_dir = os.path.join(download_dir, 'cef') # Delete the existing CEF directory if requested. if options.forceclean and os.path.exists(cef_dir): delete_directory(cef_dir) # Determine the type of CEF checkout to use. -if os.path.exists(cef_dir): - if is_git_checkout(cef_dir): - cef_use_git = True - elif is_svn_checkout(cef_dir): - cef_use_git = False - else: - raise Exception("Not a valid CEF checkout: %s" % (cef_dir)) - - if cef_use_git == options.usesvn: - raise Exception( - "The existing and requested CEF checkout types do not match") -else: - cef_use_git = not options.usesvn +if os.path.exists(cef_dir) and not is_git_checkout(cef_dir): + raise Exception("Not a valid CEF Git checkout: %s" % (cef_dir)) # Determine the CEF download URL to use. if options.url == '': - if cef_branch == 'trunk': - if cef_use_git: - cef_url = cef_git_trunk_url - else: - cef_url = cef_svn_trunk_url - else: - if cef_use_git: - cef_url = cef_git_branch_url - else: - cef_url = cef_svn_branch_url - cef_url = cef_url.replace('%1', cef_branch) + cef_url = cef_git_url else: cef_url = options.url # Verify that the requested CEF URL matches the existing checkout. if os.path.exists(cef_dir): - if cef_use_git: - cef_existing_url = get_git_url(cef_dir) - else: - cef_existing_url = get_svn_info(cef_dir)['url'] + cef_existing_url = get_git_url(cef_dir) if cef_url != cef_existing_url: raise Exception('Requested CEF checkout URL %s does not match existing '+\ 'URL %s' % (cef_url, cef_existing_url)) @@ -633,77 +570,44 @@ msg("CEF Branch: %s" % (cef_branch)) msg("CEF URL: %s" % (cef_url)) msg("CEF Source Directory: %s" % (cef_dir)) -# Determine the CEF SVN revision or Git checkout to use. +# Determine the CEF Git branch to use. if options.checkout == '': - # Use the CEF head revision. - if cef_use_git: + # Target the most recent branch commit from the remote repo. + if cef_branch == 'trunk': cef_checkout = 'origin/master' else: - cef_checkout = get_svn_info(cef_url)['revision'] + cef_checkout = 'origin/' + cef_branch else: cef_checkout = options.checkout - if not cef_use_git and not cef_checkout.isdigit(): - raise Exception("Invalid SVN revision number: %s" % (cef_checkout)) # Create the CEF checkout if necessary. if not options.noupdate and not os.path.exists(cef_dir): cef_checkout_new = True - if cef_use_git: - run('%s clone %s %s' % (git_exe, cef_url, cef_dir), download_dir, \ - depot_tools_dir) - else: - run('%s checkout %s -r %s %s' % (svn_exe, cef_url, cef_checkout, cef_dir), \ - download_dir, depot_tools_dir) + run('%s clone %s %s' % (git_exe, cef_url, cef_dir), download_dir, \ + depot_tools_dir) else: cef_checkout_new = False -# Verify the CEF checkout. -if not options.dryrun: - if cef_use_git and not is_git_checkout(cef_dir): - raise Exception('Not a valid git checkout: %s' % (cef_dir)) - if not cef_use_git and not is_svn_checkout(cef_dir): - raise Exception('Not a valid svn checkout: %s' % (cef_dir)) - # Update the CEF checkout if necessary. if not options.noupdate and os.path.exists(cef_dir): - if cef_use_git: - cef_current_hash = get_git_hash(cef_dir, 'HEAD') + cef_current_hash = get_git_hash(cef_dir, 'HEAD') - if not cef_checkout_new: - # Fetch new sources. - run('%s fetch' % (git_exe), cef_dir, depot_tools_dir) + if not cef_checkout_new: + # Fetch new sources. + run('%s fetch' % (git_exe), cef_dir, depot_tools_dir) - cef_desired_hash = get_git_hash(cef_dir, cef_checkout) - cef_checkout_changed = cef_checkout_new or force_change or \ - cef_current_hash != cef_desired_hash + cef_desired_hash = get_git_hash(cef_dir, cef_checkout) + cef_checkout_changed = cef_checkout_new or force_change or \ + cef_current_hash != cef_desired_hash - msg("CEF Current Checkout: %s" % (cef_current_hash)) - msg("CEF Current Revision: %s" % \ - (get_git_svn_revision(cef_dir, cef_current_hash))) - msg("CEF Desired Checkout: %s (%s)" % (cef_desired_hash, cef_checkout)) - msg("CEF Desired Revision: %s" % \ - (get_git_svn_revision(cef_dir, cef_desired_hash))) + msg("CEF Current Checkout: %s" % (cef_current_hash)) + msg("CEF Desired Checkout: %s (%s)" % (cef_desired_hash, cef_checkout)) - if cef_checkout_changed: - # Checkout the requested branch. - run('%s checkout %s%s' % - (git_exe, ('--force ' if options.forceclean else ''), cef_checkout), \ - cef_dir, depot_tools_dir) - else: - cef_current_info = get_svn_info(cef_dir) - if cef_current_info['url'] != cef_url: - raise Exception("CEF URL does not match; found %s, expected %s" % - (cef_current_info['url'], cef_url)) - - cef_checkout_changed = cef_checkout_new or force_change or \ - cef_current_info['revision'] != cef_checkout - - msg("CEF Current Revision: %s" % (cef_current_info['revision'])) - msg("CEF Desired Revision: %s" % (cef_checkout)) - - if cef_checkout_changed and not cef_checkout_new: - # Update to the requested revision. - run('%s update -r %s' % (svn_exe, cef_checkout), cef_dir, depot_tools_dir) + if cef_checkout_changed: + # Checkout the requested branch. + run('%s checkout %s%s' % + (git_exe, ('--force ' if options.forceclean else ''), cef_checkout), \ + cef_dir, depot_tools_dir) else: cef_checkout_changed = False @@ -806,12 +710,8 @@ if not options.noupdate and os.path.exists(chromium_src_dir): chromium_current_hash != chromium_desired_hash msg("Chromium Current Checkout: %s" % (chromium_current_hash)) - msg("Chromium Current Revision: %s" % \ - (get_git_svn_revision(chromium_src_dir, chromium_current_hash))) msg("Chromium Desired Checkout: %s (%s)" % \ (chromium_desired_hash, chromium_checkout)) - msg("Chromium Desired Revision: %s" % \ - (get_git_svn_revision(chromium_src_dir, chromium_desired_hash))) else: chromium_checkout_changed = options.dryrun @@ -828,7 +728,8 @@ if options.forceclean and os.path.exists(out_src_dir): # directory. It will be moved back from the download directory later. if os.path.exists(out_src_dir): old_branch = read_branch_config_file(out_src_dir) - if chromium_checkout_changed or old_branch != cef_branch: + if old_branch != '' and (chromium_checkout_changed or \ + old_branch != cef_branch): old_out_dir = os.path.join(download_dir, 'out_' + old_branch) move_directory(out_src_dir, old_out_dir) @@ -873,8 +774,15 @@ if os.path.exists(cef_dir) and not os.path.exists(cef_src_dir): copy_directory(cef_dir, cef_src_dir) # Restore the src/out directory. -if os.path.exists(out_dir) and not os.path.exists(out_src_dir): +out_src_dir_exists = os.path.exists(out_src_dir) +if os.path.exists(out_dir) and not out_src_dir_exists: move_directory(out_dir, out_src_dir) + out_src_dir_exists = True +elif not out_src_dir_exists: + create_directory(out_src_dir) + +# Write the config file for identifying the branch. +write_branch_config_file(out_src_dir, cef_branch) ## @@ -883,7 +791,7 @@ if os.path.exists(out_dir) and not os.path.exists(out_src_dir): if not options.nobuild and (chromium_checkout_changed or \ cef_checkout_changed or options.forcebuild or \ - not os.path.exists(out_src_dir)): + not out_src_dir_exists): # Building should also force a distribution. options.forcedistrib = True @@ -906,9 +814,6 @@ if not options.nobuild and (chromium_checkout_changed or \ path = os.path.join(cef_src_dir, 'cef_create_projects'+script_ext) run(path, cef_src_dir, depot_tools_dir) - # Write the config file for identifying the branch. - write_branch_config_file(out_src_dir, cef_branch) - # Build using Ninja. command = 'ninja -C ' if options.verbosebuild: diff --git a/tools/automate/gitsvnmirror.py b/tools/automate/gitsvnmirror.py deleted file mode 100644 index 32334a407..000000000 --- a/tools/automate/gitsvnmirror.py +++ /dev/null @@ -1,241 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2013 The Chromium Embedded Framework Authors. All rights -# reserved. Use of this source code is governed by a BSD-style license that -# can be found in the LICENSE file. - -import httplib -from optparse import OptionParser -import os -import re -import shlex -import subprocess -import sys -import urllib -import urlparse - -# Cannot be loaded as a module. -if __name__ != "__main__": - sys.stderr.write('This file cannot be loaded as a module!') - sys.exit() - -def run(command_line, working_dir): - """ Run the specified command line. """ - if not options.quiet: - print '-------- Running "%s" in "%s"...' % (command_line, working_dir) - if not options.dryrun: - args = shlex.split(command_line.replace('\\', '\\\\')) - return subprocess.check_call(args, cwd=working_dir, env=os.environ, - shell=(sys.platform == 'win32')) - -def fail(message): - """ Exit the script due to an execution failure. """ - print message - sys.exit(1) - -def url_request(url, method, headers, body, expected_response): - """ Execute an arbitrary request. """ - parsed_url = urlparse.urlparse(url) - if parsed_url.scheme == "http": - connection = httplib.HTTPConnection(parsed_url.netloc) - elif parsed_url.scheme == "https": - connection = httplib.HTTPSConnection(parsed_url.netloc) - else: - print 'Unsupported URL format for %s' % url - return None - - connection.putrequest(method, url) - - if not headers is None: - for key, val in headers.iteritems(): - connection.putheader(key, val) - if not body is None: - connection.putheader('Content-Length', len(body)) - connection.endheaders() - - if not body is None: - connection.send(body) - - response = connection.getresponse() - if response.status == expected_response: - return response.read() - else: - print 'URL %s returned unexpected response code %d' % \ - (url, response.status) - return None - -def url_propfind(url, depth, body): - """ Execute a PROPFIND request. """ - return url_request(url, 'PROPFIND', - {'Depth': depth, 'Content-Type': 'text/xml'}, body, 207) - -def url_get(url): - """ Execute a GET request. """ - return url_request(url, 'GET', None, None, 200) - -def extract_string(str, start, end): - """ Returns the string between start and end. """ - s = str.find(start) - if s < 0: - return None - slen = len(start) - e = str.find(end, s + slen) - if e < 0: - return None - return str[s + slen:e] - -def extract_int(str, start, end): - """ Returns the integer between start and end. """ - val = extract_string(str, start, end) - if not val is None and re.match('^[0-9]{1,}$', val): - return int(val) - return None - -def read_file(name, normalize = True): - """ Read a file. """ - try: - f = open(name, 'r') - # read the data - data = f.read() - if normalize: - # normalize line endings - data = data.replace("\r\n", "\n") - return data - except IOError, (errno, strerror): - print 'Failed to read file %s: %s' % (name, strerror) - else: - f.close() - return None - -def write_file(name, data): - """ Write a file. """ - try: - f = open(name, 'w') - f.write(data) - return True - except IOError, (errno, strerror): - print 'Failed to write file %s: %s' % (name, strerror) - else: - f.close() - return True - -def read_cache_file(name, args): - """ Read and parse a cache file (key=value pairs, one per line). """ - content = read_file(name) - if content is None: - return False - lines = content.split("\n") - for line in lines: - parts = line.split('=', 1) - if len(parts) == 2: - args[parts[0]] = parts[1] - return True - -def write_cache_file(name, args): - """ Write a cache file (key=value pairs, one per line). """ - data = '' - for key, val in args.iteritems(): - data = data + key + '=' + str(val) + "\n" - return write_file(name, data) - - -# Parse command-line options. -disc = """This utility creates and synchronizes git-svn clones of CEF SVN -repositories.""" - -parser = OptionParser(description=disc) -parser.add_option('--storage-dir', dest='storagedir', metavar='DIR', - help='local directory where data will be stored') -parser.add_option('--branch', dest='branch', - help='CEF branch to clone ' + - '(trunk/cef3, branches/1453/cef3, etc)') -parser.add_option('--git-repo', dest='gitrepo', - help='remote repo where the git data will be pushed ' + - '(user@domain:path/to/repo.git)') -parser.add_option('--force', - action='store_true', dest='force', default=False, - help="force the run even if the revision hasn't changed") -parser.add_option('--dry-run', - action='store_true', dest='dryrun', default=False, - help="output commands without executing them") -parser.add_option('-q', '--quiet', - action='store_true', dest='quiet', default=False, - help='do not output detailed status information') -(options, args) = parser.parse_args() - -# Required options. -if options.storagedir is None or options.branch is None or \ - options.gitrepo is None: - parser.print_help(sys.stderr) - sys.exit(1) - -# Validate the git repo format. Should be user@domain:path/to/repo.git -if not re.match( - '^[a-zA-Z0-9_\-]{1,}@[a-zA-Z0-9\-\.]{1,}:[a-zA-Z0-9\-_/]{1,}\.git$', - options.gitrepo): - fail('Invalid git repo format: %s' % options.gitrepo) - -svn_url = 'https://chromiumembedded.googlecode.com/svn/' + options.branch - -# Verify that the requested branch is valid CEF root directory. -value = url_get(svn_url + '/CHROMIUM_BUILD_COMPATIBILITY.txt') -if value is None: - fail('Invalid branch "%s"' % options.branch) - -# Retrieve the most recent revision for the branch. -revision = None -request = '' + \ - '' -value = url_propfind(svn_url, 0, request) -if not value is None: - revision = extract_int(value, '', '') -if revision is None: - fail('Failed to discover revision for branch "%s"' % options.branch) - -branch_path_comp = options.branch.replace('/', '-') - -# Create the branch storage directory if it doesn't already exist. -branch_dir = os.path.join(options.storagedir, branch_path_comp) -if not os.path.exists(branch_dir): - os.makedirs(branch_dir) - -# Default cache configuration. -cache_config = { - 'last_revision': 0, -} - -# Create the authors.txt file if it doesn't already exist -authors_file_path = os.path.join(options.storagedir, 'authors.txt') -if not os.path.exists(authors_file_path): - content = 'magreenblatt@gmail.com = ' + \ - 'Marshall Greenblatt ' - if not write_file(authors_file_path, content): - fail('Failed to create authors.txt file: %s' % authors_file_path) - -# Read the cache file if it exists. -cache_file_path = os.path.join(branch_dir, 'cache.txt') -if os.path.exists(cache_file_path): - if not read_cache_file(cache_file_path, cache_config): - print 'Failed to read cache.txt file %s' % cache_file_path - -# Check if the revision has changed. -if not options.force and int(cache_config['last_revision']) == revision: - if not options.quiet: - print 'Already at revision %d' % revision - sys.exit() - -repo_dir = os.path.join(branch_dir, branch_path_comp) -if not os.path.exists(repo_dir): - # Create the git repository. - run('git svn clone -A %s %s %s' % (authors_file_path, svn_url, repo_dir), - branch_dir) - run('git remote add origin %s' % options.gitrepo, repo_dir) -else: - # Rebase the git repository. - run('git svn rebase --fetch-all -A %s' % authors_file_path, repo_dir) - -run('git push origin --all', repo_dir) - -# Write the cache file. -cache_config['last_revision'] = revision -if not write_cache_file(cache_file_path, cache_config): - print 'Failed to write cache file %s' % cache_file_path