Improvements to automate-git.py

- Remove support for SVN checkouts (issue #1577).
- Use new Bitbucket cef.git repo for Git checkouts (issue #1577).
- Windows: Set DEPOT_TOOLS_WIN_TOOLCHAIN=0 to avoid errors while
  running Chromium hooks (issue #1533).
- Fix create/restore of out_<branch> directories.
This commit is contained in:
Marshall Greenblatt 2015-03-14 23:27:08 -04:00
parent 36a97330bd
commit 236d419d9d
2 changed files with 49 additions and 385 deletions

View File

@ -10,7 +10,6 @@ import subprocess
import sys
import tempfile
import urllib
import xml.etree.ElementTree as ET
import zipfile
##
@ -20,10 +19,7 @@ import zipfile
depot_tools_url = 'https://chromium.googlesource.com/chromium/tools/depot_tools.git'
depot_tools_archive_url = 'https://src.chromium.org/svn/trunk/tools/depot_tools.zip'
cef_git_trunk_url = 'https://chromiumembedded@bitbucket.org/chromiumembedded/trunk-cef3.git'
cef_git_branch_url = 'https://chromiumembedded@bitbucket.org/chromiumembedded/branches-%1-cef3.git'
cef_svn_trunk_url = 'https://chromiumembedded.googlecode.com/svn/trunk/cef3'
cef_svn_branch_url = 'https://chromiumembedded.googlecode.com/svn/branches/%1/cef3'
cef_git_url = 'https://chromiumembedded@bitbucket.org/chromiumembedded/cef.git'
##
@ -102,10 +98,6 @@ def is_git_checkout(path):
""" Returns true if the path represents a git checkout. """
return os.path.exists(os.path.join(path, '.git'))
def is_svn_checkout(path):
""" Returns true if the path represents an svn checkout. """
return os.path.exists(os.path.join(path, '.svn'))
def exec_cmd(cmd, path):
""" Execute the specified command and return the result. """
out = ''
@ -140,36 +132,6 @@ def get_git_url(path):
return result['out'].strip()
return 'Unknown'
def get_git_svn_revision(path, branch):
""" Returns the SVN revision associated with the specified path and git
branch/tag/hash. """
svn_rev = "None"
cmd = "%s log --grep=^git-svn-id: -n 1 %s" % (git_exe, branch)
result = exec_cmd(cmd, path)
if result['err'] == '':
for line in result['out'].split('\n'):
if line.find("git-svn-id") > 0:
svn_rev = line.split("@")[1].split()[0]
break
return svn_rev
def get_svn_info(path):
""" Retrieves the URL and revision from svn info. """
url = 'None'
rev = 'None'
cmd = "%s info --xml %s" % (svn_exe, path)
is_http = path[0:4] == 'http'
if is_http or os.path.exists(path):
result = exec_cmd(cmd, path if not is_http else '.')
if result['err'] == '':
tree = ET.ElementTree(ET.fromstring(result['out']))
entry = tree.getroot().find('entry')
url = entry.find('url').text
rev = entry.attrib['revision']
else:
raise Exception("Failed to execute svn info: %s" % (result['err']))
return {'url': url, 'revision': rev}
def download_and_extract(src, target, contents_prefix):
""" Extracts the contents of src, which may be a URL or local file, to the
target directory. """
@ -332,17 +294,12 @@ parser.add_option('--branch', dest='branch',
default='trunk')
parser.add_option('--url', dest='url',
help='CEF download URL. If not specified the default URL '+\
'will be used for the chosen branch.',
'will be used.',
default='')
parser.add_option('--checkout', dest='checkout',
help='Version of CEF to checkout. If not specified the '+\
'most recent version of the branch will be used. '+\
'If --use-git is specified this should be a Git '+\
'branch/hash/tag instead of an SVN revision number.',
'most recent remote version of the branch will be used.',
default='')
parser.add_option('--use-svn',
action='store_true', dest='usesvn', default=False,
help="Download CEF source code using SVN instead of Git.")
parser.add_option('--chromium-checkout', dest='chromiumcheckout',
help='Version of Chromium to checkout (Git '+\
'branch/hash/tag). This overrides the value specified '+\
@ -496,6 +453,10 @@ if options.branch != 'trunk' and not options.branch.isdigit():
print 'Invalid branch value: %s' % (options.branch)
cef_branch = options.branch
if cef_branch != 'trunk' and int(cef_branch) <= 1453:
print 'The requested branch is too old to build using this tool'
sys.exit()
# True if the requested branch is 2272 or newer.
branch_is_2272_or_newer = (cef_branch == 'trunk' or int(cef_branch) >= 2272)
@ -512,6 +473,10 @@ gyp_needs_target_arch_x64 = options.x64build and \
# Options that force the sources to change.
force_change = options.forceclean or options.forceupdate
if platform == 'windows':
# Avoid errors when the "vs_toolchain.py update" Chromium hook runs.
os.environ['DEPOT_TOOLS_WIN_TOOLCHAIN'] = '0'
##
# Manage the download directory.
@ -561,20 +526,16 @@ if not options.noupdate:
else:
run('update_depot_tools', depot_tools_dir, depot_tools_dir);
# Determine the svn/git executables to use.
# Determine the git executables to use.
if platform == 'windows':
# Force use of the version bundled with depot_tools.
svn_exe = os.path.join(depot_tools_dir, 'svn.bat')
git_exe = os.path.join(depot_tools_dir, 'git.bat')
if options.dryrun and (not os.path.exists(svn_exe) or \
not os.path.exists(git_exe)):
if options.dryrun and not os.path.exists(git_exe):
sys.stdout.write("WARNING: --dry-run assumes that depot_tools" \
" is already in your PATH. If it isn't\nplease" \
" specify a --depot-tools-dir value.\n")
svn_exe = 'svn.bat'
git_exe = 'git.bat'
else:
svn_exe = 'svn'
git_exe = 'git'
@ -582,49 +543,25 @@ else:
# Manage the cef directory.
##
cef_dir = os.path.join(download_dir, 'cef_' + cef_branch)
cef_dir = os.path.join(download_dir, 'cef')
# Delete the existing CEF directory if requested.
if options.forceclean and os.path.exists(cef_dir):
delete_directory(cef_dir)
# Determine the type of CEF checkout to use.
if os.path.exists(cef_dir):
if is_git_checkout(cef_dir):
cef_use_git = True
elif is_svn_checkout(cef_dir):
cef_use_git = False
else:
raise Exception("Not a valid CEF checkout: %s" % (cef_dir))
if cef_use_git == options.usesvn:
raise Exception(
"The existing and requested CEF checkout types do not match")
else:
cef_use_git = not options.usesvn
if os.path.exists(cef_dir) and not is_git_checkout(cef_dir):
raise Exception("Not a valid CEF Git checkout: %s" % (cef_dir))
# Determine the CEF download URL to use.
if options.url == '':
if cef_branch == 'trunk':
if cef_use_git:
cef_url = cef_git_trunk_url
else:
cef_url = cef_svn_trunk_url
else:
if cef_use_git:
cef_url = cef_git_branch_url
else:
cef_url = cef_svn_branch_url
cef_url = cef_url.replace('%1', cef_branch)
cef_url = cef_git_url
else:
cef_url = options.url
# Verify that the requested CEF URL matches the existing checkout.
if os.path.exists(cef_dir):
if cef_use_git:
cef_existing_url = get_git_url(cef_dir)
else:
cef_existing_url = get_svn_info(cef_dir)['url']
if cef_url != cef_existing_url:
raise Exception('Requested CEF checkout URL %s does not match existing '+\
'URL %s' % (cef_url, cef_existing_url))
@ -633,40 +570,26 @@ msg("CEF Branch: %s" % (cef_branch))
msg("CEF URL: %s" % (cef_url))
msg("CEF Source Directory: %s" % (cef_dir))
# Determine the CEF SVN revision or Git checkout to use.
# Determine the CEF Git branch to use.
if options.checkout == '':
# Use the CEF head revision.
if cef_use_git:
# Target the most recent branch commit from the remote repo.
if cef_branch == 'trunk':
cef_checkout = 'origin/master'
else:
cef_checkout = get_svn_info(cef_url)['revision']
cef_checkout = 'origin/' + cef_branch
else:
cef_checkout = options.checkout
if not cef_use_git and not cef_checkout.isdigit():
raise Exception("Invalid SVN revision number: %s" % (cef_checkout))
# Create the CEF checkout if necessary.
if not options.noupdate and not os.path.exists(cef_dir):
cef_checkout_new = True
if cef_use_git:
run('%s clone %s %s' % (git_exe, cef_url, cef_dir), download_dir, \
depot_tools_dir)
else:
run('%s checkout %s -r %s %s' % (svn_exe, cef_url, cef_checkout, cef_dir), \
download_dir, depot_tools_dir)
else:
cef_checkout_new = False
# Verify the CEF checkout.
if not options.dryrun:
if cef_use_git and not is_git_checkout(cef_dir):
raise Exception('Not a valid git checkout: %s' % (cef_dir))
if not cef_use_git and not is_svn_checkout(cef_dir):
raise Exception('Not a valid svn checkout: %s' % (cef_dir))
# Update the CEF checkout if necessary.
if not options.noupdate and os.path.exists(cef_dir):
if cef_use_git:
cef_current_hash = get_git_hash(cef_dir, 'HEAD')
if not cef_checkout_new:
@ -678,32 +601,13 @@ if not options.noupdate and os.path.exists(cef_dir):
cef_current_hash != cef_desired_hash
msg("CEF Current Checkout: %s" % (cef_current_hash))
msg("CEF Current Revision: %s" % \
(get_git_svn_revision(cef_dir, cef_current_hash)))
msg("CEF Desired Checkout: %s (%s)" % (cef_desired_hash, cef_checkout))
msg("CEF Desired Revision: %s" % \
(get_git_svn_revision(cef_dir, cef_desired_hash)))
if cef_checkout_changed:
# Checkout the requested branch.
run('%s checkout %s%s' %
(git_exe, ('--force ' if options.forceclean else ''), cef_checkout), \
cef_dir, depot_tools_dir)
else:
cef_current_info = get_svn_info(cef_dir)
if cef_current_info['url'] != cef_url:
raise Exception("CEF URL does not match; found %s, expected %s" %
(cef_current_info['url'], cef_url))
cef_checkout_changed = cef_checkout_new or force_change or \
cef_current_info['revision'] != cef_checkout
msg("CEF Current Revision: %s" % (cef_current_info['revision']))
msg("CEF Desired Revision: %s" % (cef_checkout))
if cef_checkout_changed and not cef_checkout_new:
# Update to the requested revision.
run('%s update -r %s' % (svn_exe, cef_checkout), cef_dir, depot_tools_dir)
else:
cef_checkout_changed = False
@ -806,12 +710,8 @@ if not options.noupdate and os.path.exists(chromium_src_dir):
chromium_current_hash != chromium_desired_hash
msg("Chromium Current Checkout: %s" % (chromium_current_hash))
msg("Chromium Current Revision: %s" % \
(get_git_svn_revision(chromium_src_dir, chromium_current_hash)))
msg("Chromium Desired Checkout: %s (%s)" % \
(chromium_desired_hash, chromium_checkout))
msg("Chromium Desired Revision: %s" % \
(get_git_svn_revision(chromium_src_dir, chromium_desired_hash)))
else:
chromium_checkout_changed = options.dryrun
@ -828,7 +728,8 @@ if options.forceclean and os.path.exists(out_src_dir):
# directory. It will be moved back from the download directory later.
if os.path.exists(out_src_dir):
old_branch = read_branch_config_file(out_src_dir)
if chromium_checkout_changed or old_branch != cef_branch:
if old_branch != '' and (chromium_checkout_changed or \
old_branch != cef_branch):
old_out_dir = os.path.join(download_dir, 'out_' + old_branch)
move_directory(out_src_dir, old_out_dir)
@ -873,8 +774,15 @@ if os.path.exists(cef_dir) and not os.path.exists(cef_src_dir):
copy_directory(cef_dir, cef_src_dir)
# Restore the src/out directory.
if os.path.exists(out_dir) and not os.path.exists(out_src_dir):
out_src_dir_exists = os.path.exists(out_src_dir)
if os.path.exists(out_dir) and not out_src_dir_exists:
move_directory(out_dir, out_src_dir)
out_src_dir_exists = True
elif not out_src_dir_exists:
create_directory(out_src_dir)
# Write the config file for identifying the branch.
write_branch_config_file(out_src_dir, cef_branch)
##
@ -883,7 +791,7 @@ if os.path.exists(out_dir) and not os.path.exists(out_src_dir):
if not options.nobuild and (chromium_checkout_changed or \
cef_checkout_changed or options.forcebuild or \
not os.path.exists(out_src_dir)):
not out_src_dir_exists):
# Building should also force a distribution.
options.forcedistrib = True
@ -906,9 +814,6 @@ if not options.nobuild and (chromium_checkout_changed or \
path = os.path.join(cef_src_dir, 'cef_create_projects'+script_ext)
run(path, cef_src_dir, depot_tools_dir)
# Write the config file for identifying the branch.
write_branch_config_file(out_src_dir, cef_branch)
# Build using Ninja.
command = 'ninja -C '
if options.verbosebuild:

View File

@ -1,241 +0,0 @@
#!/usr/bin/env python
# Copyright (c) 2013 The Chromium Embedded Framework Authors. All rights
# reserved. Use of this source code is governed by a BSD-style license that
# can be found in the LICENSE file.
import httplib
from optparse import OptionParser
import os
import re
import shlex
import subprocess
import sys
import urllib
import urlparse
# Cannot be loaded as a module.
if __name__ != "__main__":
sys.stderr.write('This file cannot be loaded as a module!')
sys.exit()
def run(command_line, working_dir):
""" Run the specified command line. """
if not options.quiet:
print '-------- Running "%s" in "%s"...' % (command_line, working_dir)
if not options.dryrun:
args = shlex.split(command_line.replace('\\', '\\\\'))
return subprocess.check_call(args, cwd=working_dir, env=os.environ,
shell=(sys.platform == 'win32'))
def fail(message):
""" Exit the script due to an execution failure. """
print message
sys.exit(1)
def url_request(url, method, headers, body, expected_response):
""" Execute an arbitrary request. """
parsed_url = urlparse.urlparse(url)
if parsed_url.scheme == "http":
connection = httplib.HTTPConnection(parsed_url.netloc)
elif parsed_url.scheme == "https":
connection = httplib.HTTPSConnection(parsed_url.netloc)
else:
print 'Unsupported URL format for %s' % url
return None
connection.putrequest(method, url)
if not headers is None:
for key, val in headers.iteritems():
connection.putheader(key, val)
if not body is None:
connection.putheader('Content-Length', len(body))
connection.endheaders()
if not body is None:
connection.send(body)
response = connection.getresponse()
if response.status == expected_response:
return response.read()
else:
print 'URL %s returned unexpected response code %d' % \
(url, response.status)
return None
def url_propfind(url, depth, body):
""" Execute a PROPFIND request. """
return url_request(url, 'PROPFIND',
{'Depth': depth, 'Content-Type': 'text/xml'}, body, 207)
def url_get(url):
""" Execute a GET request. """
return url_request(url, 'GET', None, None, 200)
def extract_string(str, start, end):
""" Returns the string between start and end. """
s = str.find(start)
if s < 0:
return None
slen = len(start)
e = str.find(end, s + slen)
if e < 0:
return None
return str[s + slen:e]
def extract_int(str, start, end):
""" Returns the integer between start and end. """
val = extract_string(str, start, end)
if not val is None and re.match('^[0-9]{1,}$', val):
return int(val)
return None
def read_file(name, normalize = True):
""" Read a file. """
try:
f = open(name, 'r')
# read the data
data = f.read()
if normalize:
# normalize line endings
data = data.replace("\r\n", "\n")
return data
except IOError, (errno, strerror):
print 'Failed to read file %s: %s' % (name, strerror)
else:
f.close()
return None
def write_file(name, data):
""" Write a file. """
try:
f = open(name, 'w')
f.write(data)
return True
except IOError, (errno, strerror):
print 'Failed to write file %s: %s' % (name, strerror)
else:
f.close()
return True
def read_cache_file(name, args):
""" Read and parse a cache file (key=value pairs, one per line). """
content = read_file(name)
if content is None:
return False
lines = content.split("\n")
for line in lines:
parts = line.split('=', 1)
if len(parts) == 2:
args[parts[0]] = parts[1]
return True
def write_cache_file(name, args):
""" Write a cache file (key=value pairs, one per line). """
data = ''
for key, val in args.iteritems():
data = data + key + '=' + str(val) + "\n"
return write_file(name, data)
# Parse command-line options.
disc = """This utility creates and synchronizes git-svn clones of CEF SVN
repositories."""
parser = OptionParser(description=disc)
parser.add_option('--storage-dir', dest='storagedir', metavar='DIR',
help='local directory where data will be stored')
parser.add_option('--branch', dest='branch',
help='CEF branch to clone ' +
'(trunk/cef3, branches/1453/cef3, etc)')
parser.add_option('--git-repo', dest='gitrepo',
help='remote repo where the git data will be pushed ' +
'(user@domain:path/to/repo.git)')
parser.add_option('--force',
action='store_true', dest='force', default=False,
help="force the run even if the revision hasn't changed")
parser.add_option('--dry-run',
action='store_true', dest='dryrun', default=False,
help="output commands without executing them")
parser.add_option('-q', '--quiet',
action='store_true', dest='quiet', default=False,
help='do not output detailed status information')
(options, args) = parser.parse_args()
# Required options.
if options.storagedir is None or options.branch is None or \
options.gitrepo is None:
parser.print_help(sys.stderr)
sys.exit(1)
# Validate the git repo format. Should be user@domain:path/to/repo.git
if not re.match(
'^[a-zA-Z0-9_\-]{1,}@[a-zA-Z0-9\-\.]{1,}:[a-zA-Z0-9\-_/]{1,}\.git$',
options.gitrepo):
fail('Invalid git repo format: %s' % options.gitrepo)
svn_url = 'https://chromiumembedded.googlecode.com/svn/' + options.branch
# Verify that the requested branch is valid CEF root directory.
value = url_get(svn_url + '/CHROMIUM_BUILD_COMPATIBILITY.txt')
if value is None:
fail('Invalid branch "%s"' % options.branch)
# Retrieve the most recent revision for the branch.
revision = None
request = '<?xml version="1.0" encoding="utf-8"?><propfind xmlns="DAV:">' + \
'<prop><version-name xmlns="DAV:"/></prop></propfind>'
value = url_propfind(svn_url, 0, request)
if not value is None:
revision = extract_int(value, '<lp1:version-name>', '</lp1:version-name>')
if revision is None:
fail('Failed to discover revision for branch "%s"' % options.branch)
branch_path_comp = options.branch.replace('/', '-')
# Create the branch storage directory if it doesn't already exist.
branch_dir = os.path.join(options.storagedir, branch_path_comp)
if not os.path.exists(branch_dir):
os.makedirs(branch_dir)
# Default cache configuration.
cache_config = {
'last_revision': 0,
}
# Create the authors.txt file if it doesn't already exist
authors_file_path = os.path.join(options.storagedir, 'authors.txt')
if not os.path.exists(authors_file_path):
content = 'magreenblatt@gmail.com = ' + \
'Marshall Greenblatt <magreenblatt@gmail.com>'
if not write_file(authors_file_path, content):
fail('Failed to create authors.txt file: %s' % authors_file_path)
# Read the cache file if it exists.
cache_file_path = os.path.join(branch_dir, 'cache.txt')
if os.path.exists(cache_file_path):
if not read_cache_file(cache_file_path, cache_config):
print 'Failed to read cache.txt file %s' % cache_file_path
# Check if the revision has changed.
if not options.force and int(cache_config['last_revision']) == revision:
if not options.quiet:
print 'Already at revision %d' % revision
sys.exit()
repo_dir = os.path.join(branch_dir, branch_path_comp)
if not os.path.exists(repo_dir):
# Create the git repository.
run('git svn clone -A %s %s %s' % (authors_file_path, svn_url, repo_dir),
branch_dir)
run('git remote add origin %s' % options.gitrepo, repo_dir)
else:
# Rebase the git repository.
run('git svn rebase --fetch-all -A %s' % authors_file_path, repo_dir)
run('git push origin --all', repo_dir)
# Write the cache file.
cache_config['last_revision'] = revision
if not write_cache_file(cache_file_path, cache_config):
print 'Failed to write cache file %s' % cache_file_path