cef/tools/cefbuilds/cef_json_builder.py

483 lines
17 KiB
Python

# Copyright (c) 2016 The Chromium Embedded Framework Authors. All rights
# reserved. Use of this source code is governed by a BSD-style license that
# can be found in the LICENSE file.
import datetime
import json
import os
import re
import urllib
# Class used to build the cefbuilds JSON file. See cef_json_builder_example.py
# for example usage. See cef_json_builder_test.py for unit tests.
#
# Example JSON structure:
# {
# "linux32": {
# "versions": [
# {
# "cef_version": "3.2704.1414.g185cd6c",
# "chromium_version": "51.0.2704.47"
# "files": [
# {
# "last_modified": "2016-05-18T22:42:14.066Z"
# "name": "cef_binary_3.2704.1414.g185cd6c_linux32.tar.bz2",
# "sha1": "47c5cfea43912a1d1771f343de35b205f388415f"
# "size": "48549450",
# "type": "standard",
# }, ...
# ],
# }, ...
# ]
# }, ...
# }
#
# Notes:
# - "files" in a given version will be sorted from newest to oldest based on the
# "last_modified" value.
# - "versions" in a given platform will be sorted from newest to oldest based on
# the "last_modified" value of the first (newest) "file" sub-value.
# - There will be at most one record at the "files" level for each "type".
# This date format intentionally matches the format used in Artifactory
# directory listings.
_CEF_DATE_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
def parse_date(date):
return datetime.datetime.strptime(date, _CEF_DATE_FORMAT)
def format_date(date):
return date.strftime(_CEF_DATE_FORMAT)
# Helpers to format datetime values on JSON read/write.
def cef_from_json(json_object):
if 'last_modified' in json_object:
json_object['last_modified'] = parse_date(json_object['last_modified'])
return json_object
class cef_json_encoder(json.JSONEncoder):
def default(self, o):
if isinstance(o, datetime.datetime):
return format_date(o)
return o
_chromium_version_regex = '[1-9]{1}[0-9]{1,2}\.0\.[1-9]{1}[0-9]{2,4}\.(0|[1-9]{1}[0-9]{0,2})'
_cef_hash_regex = 'g[0-9a-f]{7}'
_cef_number_regex = '[0-9]{1,5}\.[0-9]{1,5}\.[0-9]{1,5}'
# Example: 3.2704.1414.g185cd6c
_cef_old_version_regex = _cef_number_regex + '\.' + _cef_hash_regex
# Example: 74.0.1+g62d140e+chromium-74.0.3729.6
_cef_version_release_regex = _cef_number_regex + '\+' + _cef_hash_regex + '\+chromium\-' + _chromium_version_regex
# Example: 74.0.0-master.1920+g725ed88+chromium-74.0.3729.0
_cef_version_dev_regex = _cef_number_regex + '\-\w+\.[0-9]{1,7}\+' + _cef_hash_regex + '\+chromium\-' + _chromium_version_regex
class cef_json_builder:
""" Class used to build the cefbuilds JSON file. """
def __init__(self, prettyprint=False, silent=True):
""" Create a new cef_json_builder object. """
self._prettyprint = prettyprint
self._silent = silent
self._fatalerrors = False
self.clear()
@staticmethod
def get_platforms():
""" Returns the list of supported platforms. """
return ('linux32', 'linux64', 'linuxarm', 'linuxarm64', 'macosx64',
'windows32', 'windows64')
@staticmethod
def get_distrib_types():
""" Returns the list of supported distribution types. """
return ('standard', 'minimal', 'client', 'release_symbols', 'debug_symbols')
@staticmethod
def is_valid_version(version):
""" Returns true if the specified CEF version is fully qualified and valid. """
if version is None:
return False
return bool(re.compile('^' + _cef_old_version_regex + '$').match(version)) \
or bool(re.compile('^' + _cef_version_release_regex + '$').match(version)) \
or bool(re.compile('^' + _cef_version_dev_regex + '$').match(version))
@staticmethod
def is_valid_chromium_version(version):
""" Returns true if the specified Chromium version is fully qualified and valid. """
if version is None:
return False
return version == 'master' or \
bool(re.compile('^' + _chromium_version_regex + '$').match(version))
@staticmethod
def get_file_name(version, platform, type):
""" Returns the expected distribution file name excluding extension based on
the input parameters. """
if type != 'standard':
type_str = '_' + type
else:
type_str = ''
return 'cef_binary_%s_%s%s' % (version, platform, type_str)
def clear(self):
""" Clear the contents of this object. """
self._data = {}
for platform in self.get_platforms():
self._data[platform] = {'versions': []}
self._versions = {}
self._queryct = 0
def __repr__(self):
# Return a string representation of this object.
self._sort_versions()
if self._prettyprint:
return json.dumps(
self._data,
cls=cef_json_encoder,
sort_keys=True,
indent=2,
separators=(',', ': '))
else:
return json.dumps(self._data, cls=cef_json_encoder, sort_keys=True)
def _print(self, msg):
if self._fatalerrors:
raise Exception(msg)
if not self._silent:
print msg
def get_query_count(self):
""" Returns the number of queries sent while building. """
return self._queryct
def _query_chromium_version(self, cef_version):
""" Try to remotely query the Chromium version for old-style CEF version
numbers. """
chromium_version = 'master'
git_hash = cef_version[-7:]
query_url = 'https://bitbucket.org/chromiumembedded/cef/raw/%s/CHROMIUM_BUILD_COMPATIBILITY.txt' % git_hash
self._queryct = self._queryct + 1
if not self._silent:
print 'Reading %s' % query_url
try:
# Read the remote URL contents.
handle = urllib.urlopen(query_url)
compat_value = handle.read().strip()
handle.close()
# Parse the contents.
config = eval(compat_value, {'__builtins__': None}, None)
if not 'chromium_checkout' in config:
raise Exception('Unexpected contents')
val = config['chromium_checkout']
if val.find('refs/tags/') == 0:
chromium_version = val[10:]
except Exception, e:
print 'Failed to read Chromium version information'
raise
return chromium_version
def set_chromium_version(self, cef_version, chromium_version=None):
""" Set the matching Chromium version. If the specified Chromium version is
invalid then it will be queried remotely. """
if not self.is_valid_version(cef_version):
raise Exception('Invalid CEF version: %s' % cef_version)
if not self.is_valid_chromium_version(chromium_version):
if cef_version in self._versions:
# Keep the Chromium version that we already know about.
return self._versions[cef_version]
if cef_version.find('+chromium') > 0:
# New-style CEF version numbers include the Chromium version number.
# Example: 74.0.1+g62d140e+chromium-74.0.3729.6
chromium_version = cef_version[cef_version.rfind('-') + 1:]
else:
chromium_version = self._query_chromium_version(cef_version)
if not self.is_valid_chromium_version(chromium_version):
raise Exception('Invalid Chromium version: %s' % chromium_version)
self._versions[cef_version] = chromium_version
return chromium_version
def get_chromium_version(self, cef_version):
""" Return the matching Chromium version. If not currently known it will
be parsed from the CEF version or queried remotely. """
if cef_version in self._versions:
return self._versions[cef_version]
# Identify the Chromium version.
return self.set_chromium_version(cef_version)
def has_chromium_version(self, cef_version):
""" Return True if a matching Chromium version is known. """
return cef_version in self._versions
def load(self, json_string, fatalerrors=True):
""" Load new JSON into this object. Any existing contents will be cleared.
If |fatalerrors| is True then any errors while loading the JSON file
will cause an Exception to be thrown. Otherwise, malformed entries will
will be discarded. Unrecognized keys will always be discarded silently.
"""
self.clear()
self._fatalerrors = fatalerrors
new_data = json.JSONDecoder(object_hook=cef_from_json).decode(json_string)
# Validate the new data's structure.
for platform in self._data.keys():
if not platform in new_data:
self._print('load: Platform %s not found' % platform)
continue
if not 'versions' in new_data[platform]:
self._print('load: Missing platform key(s) for %s' % platform)
continue
valid_versions = []
for version in new_data[platform]['versions']:
if not 'cef_version' in version or \
not 'chromium_version' in version or \
not 'files' in version:
self._print('load: Missing version key(s) for %s' % platform)
continue
valid_files = []
found_types = []
for file in version['files']:
if not 'type' in file or \
not 'name' in file or \
not 'size' in file or \
not 'last_modified' in file or \
not 'sha1' in file:
self._print('load: Missing file key(s) for %s %s' %
(platform, version['cef_version']))
continue
(expected_platform, expected_version,
expected_type) = self._parse_name(file['name'])
if expected_platform != platform or \
expected_version != version['cef_version'] or \
expected_type != file['type']:
self._print('load: File name/attribute mismatch for %s %s %s' %
(platform, version['cef_version'], file['name']))
continue
self._validate_args(platform, version['cef_version'], file['type'],
file['size'], file['last_modified'], file['sha1'])
if file['type'] in found_types:
self._print('load: Duplicate %s type for %s %s' %
(file['type'], platform, version['cef_version']))
continue
found_types.append(file['type'])
valid_files.append({
'type': file['type'],
'name': file['name'],
'size': file['size'],
'last_modified': file['last_modified'],
'sha1': file['sha1'],
})
if len(valid_files) > 0:
valid_versions.append({
'cef_version':
version['cef_version'],
'chromium_version':
self.set_chromium_version(version['cef_version'],
version['chromium_version']),
'files':
self._sort_files(valid_files)
})
if len(valid_versions) > 0:
self._data[platform]['versions'] = valid_versions
self._fatalerrors = False
def _sort_versions(self):
# Sort version records by first (newest) file last_modified value.
for platform in self._data.keys():
for i in range(0, len(self._data[platform]['versions'])):
self._data[platform]['versions'] = \
sorted(self._data[platform]['versions'],
key=lambda k: k['files'][0]['last_modified'],
reverse=True)
@staticmethod
def _sort_files(files):
# Sort file records by last_modified.
return sorted(files, key=lambda k: k['last_modified'], reverse=True)
@staticmethod
def _parse_name(name):
# Remove file extension.
name_no_ext = os.path.splitext(name)[0]
if name_no_ext[-4:] == '.tar':
name_no_ext = name_no_ext[:-4]
name_parts = name_no_ext.split('_')
if len(
name_parts) < 4 or name_parts[0] != 'cef' or name_parts[1] != 'binary':
raise Exception('Invalid filename: %s' % name)
# Remove 'cef' and 'binary'.
del name_parts[0]
del name_parts[0]
type = None
# Might be '<version>_<platform>_[debug|release]_symbols'.
if name_parts[-1] == 'symbols':
del name_parts[-1]
if name_parts[-1] == 'debug' or name_parts[-1] == 'release':
type = name_parts[-1] + '_symbols'
del name_parts[-1]
# Might be '<version>_<platform>_minimal'.
if name_parts[-1] == 'minimal':
type = 'minimal'
del name_parts[-1]
# Might be '<version>_<platform>_client'.
if name_parts[-1] == 'client':
type = 'client'
del name_parts[-1]
# Remainder must be '<version>_<platform>'.
if len(name_parts) != 2:
raise Exception('Invalid filename: %s' % name)
if type is None:
type = 'standard'
version = name_parts[0]
platform = name_parts[1]
return [platform, version, type]
@staticmethod
def _validate_args(platform, version, type, size, last_modified, sha1):
# Validate input arguments.
if not platform in cef_json_builder.get_platforms():
raise Exception('Unsupported platform: %s' % platform)
if not cef_json_builder.is_valid_version(version):
raise Exception('Invalid version: %s' % version)
if not type in cef_json_builder.get_distrib_types():
raise Exception('Unsupported distribution type: %s' % type)
if int(size) <= 0:
raise Exception('Invalid size: %s' % size)
if not isinstance(last_modified, datetime.datetime):
# datetime will throw a ValueException if it doesn't parse.
parse_date(last_modified)
if not re.compile('^[0-9a-f]{40}$').match(sha1):
raise Exception('Invalid sha1: %s' % sha1)
def add_file(self, name, size, last_modified, sha1):
""" Add a file record with the specified attributes. Returns True if the
file is added or False if a file with the same |name| and |sha1|
already exists. """
# Parse the file name.
(platform, version, type) = self._parse_name(name)
if not isinstance(size, (int, long)):
size = int(size)
if not isinstance(last_modified, datetime.datetime):
last_modified = parse_date(last_modified)
# Validate arguments.
self._validate_args(platform, version, type, size, last_modified, sha1)
# Find the existing version record.
version_idx = -1
for i in range(0, len(self._data[platform]['versions'])):
if self._data[platform]['versions'][i]['cef_version'] == version:
# Check the version record.
self._print('add_file: Check %s %s' % (platform, version))
version_idx = i
break
if version_idx == -1:
# Add a new version record.
self._print('add_file: Add %s %s' % (platform, version))
self._data[platform]['versions'].append({
'cef_version': version,
'chromium_version': self.get_chromium_version(version),
'files': []
})
version_idx = len(self._data[platform]['versions']) - 1
# Find the existing file record with matching type.
file_changed = True
for i in range(0,
len(self._data[platform]['versions'][version_idx]['files'])):
if self._data[platform]['versions'][version_idx]['files'][i][
'type'] == type:
existing_sha1 = self._data[platform]['versions'][version_idx]['files'][
i]['sha1']
if existing_sha1 != sha1:
# Remove the existing file record.
self._print(' Remove %s %s' % (name, existing_sha1))
del self._data[platform]['versions'][version_idx]['files'][i]
else:
file_changed = False
break
if file_changed:
# Add a new file record.
self._print(' Add %s %s' % (name, sha1))
self._data[platform]['versions'][version_idx]['files'].append({
'type': type,
'name': name,
'size': size,
'last_modified': last_modified,
'sha1': sha1
})
# Sort file records by last_modified.
# This is necessary for _sort_versions() to function correctly.
self._data[platform]['versions'][version_idx]['files'] = \
self._sort_files(self._data[platform]['versions'][version_idx]['files'])
return file_changed
def get_files(self, platform=None, version=None, type=None):
""" Return the files that match the input parameters.
All parameters are optional. Version will do partial matching. """
results = []
if platform is None:
platforms = self._data.keys()
else:
platforms = [platform]
for platform in platforms:
for version_obj in self._data[platform]['versions']:
if version is None or version_obj['cef_version'].find(version) == 0:
for file_obj in version_obj['files']:
if type is None or type == file_obj['type']:
result_obj = file_obj
# Add additional metadata.
result_obj['platform'] = platform
result_obj['cef_version'] = version_obj['cef_version']
result_obj['chromium_version'] = version_obj['chromium_version']
results.append(result_obj)
return results
def get_versions(self, platform):
""" Return all versions for the specified |platform|. """
return self._data[platform]['versions']