From 769150e178fbb6b04a02ec7130972f8a5e931254 Mon Sep 17 00:00:00 2001 From: Marshall Greenblatt Date: Fri, 27 May 2016 17:53:18 -0400 Subject: [PATCH] Add cefbuilds tools. See AutomatedBuildSetup Wiki page for usage. --- tools/cefbuilds/cef_html_builder.py | 283 +++++++++++++ tools/cefbuilds/cef_json_builder.py | 434 ++++++++++++++++++++ tools/cefbuilds/cef_json_builder_example.py | 144 +++++++ tools/cefbuilds/cef_json_builder_test.py | 318 ++++++++++++++ tools/cefbuilds/index.html.in | 178 ++++++++ 5 files changed, 1357 insertions(+) create mode 100644 tools/cefbuilds/cef_html_builder.py create mode 100644 tools/cefbuilds/cef_json_builder.py create mode 100644 tools/cefbuilds/cef_json_builder_example.py create mode 100644 tools/cefbuilds/cef_json_builder_test.py create mode 100644 tools/cefbuilds/index.html.in diff --git a/tools/cefbuilds/cef_html_builder.py b/tools/cefbuilds/cef_html_builder.py new file mode 100644 index 000000000..ad2b4aa48 --- /dev/null +++ b/tools/cefbuilds/cef_html_builder.py @@ -0,0 +1,283 @@ +# Copyright (c) 2016 The Chromium Embedded Framework Authors. All rights +# reserved. Use of this source code is governed by a BSD-style license that +# can be found in the LICENSE file. + +from cef_json_builder import cef_json_builder +import datetime +import math +import os +import sys + +# Class used to build the cefbuilds HTML file. Generate an index.json file using +# the cef_json_builder_example.py tool (or some other means) and then run: +# > python cef_html_builder.py index.json index.html.in index.html +# +# Expected HTML template format is: +# +# Header +# +# $platform_name$ +# +# +# $platform_name$ Builds: +# +# CEF version $cef_version$ Files: +# +# File $file$ size $size$ sha1 $sha1$ +# +# +# +# Footer +# +# Notes: +# - The variables ("$key$") available in each section generally match the key +# names that exist in the JSON file for that section. Additional variables are +# exposed where needed. +# - Some global variables like "$year$" will be replaced in the whole template +# before further parsing occurs. + +class cef_html_builder: + """ Class used to build the cefbuilds HTML file. """ + + def __init__(self, branding=''): + """ Create a new cef_html_builder object. """ + self.clear() + self._branding = branding + + def clear(self): + """ Clear the contents of this object. """ + self._parts = {} + return; + + @staticmethod + def _token(key): + # Returns the token representation of |key| + return '$' + key + '$' + + @staticmethod + def _section_tags(section): + # Returns the start and end tags for |section| + return ('', '') + + @staticmethod + def _section_key(section): + # Returns the replacement key for |section| + return section + '_section' + + @staticmethod + def _replace(str, key, value): + # Replaces all instances of |key| with |value| in |str|. + return str.replace(cef_html_builder._token(key), value) + + @staticmethod + def _replace_all(str, dict): + for key, value in dict.iteritems(): + str = cef_html_builder._replace(str, key, value) + return str + + @staticmethod + def _extract(str, section): + # Extracts the |section| region and replaces it with a token named + # "
_section". + (start_tag, end_tag) = cef_html_builder._section_tags(section) + start_pos = str.find(start_tag) + end_pos = str.rfind(end_tag) + if start_pos < 0 or end_pos < 0: + raise Exception('Failed to find section %s' % section) + top = str[:start_pos] + middle = str[start_pos + len(start_tag):end_pos] + bottom = str[end_pos + len(end_tag):] + return (top + cef_html_builder._token(cef_html_builder._section_key(section)) + bottom, middle) + + def load(self, html_template): + """ Load the specified |html_template| string. """ + self.clear() + root = html_template + + # Extract the platform link section from root. + (root, platform_link) = self._extract(root, 'platform_link') + + # Extract platform section from root. + (root, platform) = self._extract(root, 'platform') + + # Extract version section from platform. + (platform, version) = self._extract(platform, 'version') + + # Extract file section from version. + (version, file) = self._extract(version, 'file') + + self._parts = { + 'root': root, + 'platform_link': platform_link, + 'platform': platform, + 'version': version, + 'file': file + } + + @staticmethod + def _get_platform_name(platform): + return { + 'linux32': 'Linux 32-bit', + 'linux64': 'Linux 64-bit', + 'macosx64': 'Mac OS X 64-bit', + 'windows32': 'Windows 32-bit', + 'windows64': 'Windows 64-bit' + }[platform] + + @staticmethod + def _get_type_name(type): + return { + 'standard': 'Standard Distribution', + 'minimal': 'Minimal Distribution', + 'client': 'Sample Application', + 'debug_symbols': 'Debug Symbols', + 'release_symbols': 'Release Symbols' + }[type] + + @staticmethod + def _get_date(date): + return date.strftime('%m/%d/%Y') + + @staticmethod + def _get_file_size(size): + if (size == 0): + return '0B' + size_name = ('B', 'KB', 'MB', 'GB') + i = int(math.floor(math.log(size, 1024))) + p = math.pow(1024, i) + s = round(size/p, 2) + return '%.2f %s' % (s, size_name[i]) + + @staticmethod + def _get_cef_source_url(cef_version): + branch = cef_version.split('.')[2] + return 'https://bitbucket.org/chromiumembedded/cef/get/%s.tar.bz2' % branch + + @staticmethod + def _get_chromium_source_url(chromium_version): + if chromium_version == 'master': + return 'https://chromium.googlesource.com/chromium/src.git' + return 'https://gsdview.appspot.com/chromium-browser-official/chromium-%s.tar.xz' % chromium_version + + @staticmethod + def _get_file_url(platform, cef_version, file): + return file['name'] + + @staticmethod + def _get_sha1_url(platform, cef_version, file): + return file['name'] + '.sha1' + + @staticmethod + def _get_tooltip_text(platform, cef_version, file): + return { + 'standard': 'Standard binary distribution. Includes header files, libcef_dll_wrapper source code, binary files, CMake configuration files and source code for the cefclient and cefsimple sample applications. See the included README.txt file for usage and build requirements.', + 'minimal': 'Minimal binary distribution. Includes header files, libcef_dll_wrapper source code, Release build binary files and CMake configuration files. Does not include Debug build binary files or sample application source code. See the included README.txt file for usage and build requirements.', + 'client': 'Release build of the cefclient sample application. See the included README.txt file for usage requirements.', + 'debug_symbols': 'Debug build symbols. Must be extracted and placed next to the CEF Debug binary file with the same name and version.', + 'release_symbols': 'Release build symbols. Must be extracted and placed next to the CEF Release binary file with the same name and version.' + }[file['type']] + + def generate(self, json_builder): + """ Generate HTML output based on the contents of |json_builder|. """ + if not isinstance(json_builder, cef_json_builder): + raise Exception('Invalid argument') + + # Substitution values are augmented at each nesting level. + subs = { + 'year': '2016', + 'branding': self._branding, + } + + # Substitute variables. + root_str = self._replace_all(self._parts['root'], subs) + + platform_link_strs = [] + platform_strs = [] + for platform in json_builder.get_platforms(): + subs['platform'] = platform + subs['platform_name'] = self._get_platform_name(platform) + + # Substitute variables. + platform_link_str = self._replace_all(self._parts['platform_link'], subs) + platform_str = self._replace_all(self._parts['platform'], subs) + + version_strs = [] + for version in json_builder.get_versions(platform): + subs['cef_version'] = version['cef_version'] + subs['chromium_version'] = version['chromium_version'] + subs['last_modified'] = self._get_date(version['files'][0]['last_modified']) + subs['cef_source_url'] = self._get_cef_source_url(version['cef_version']) + subs['chromium_source_url'] = self._get_chromium_source_url(version['chromium_version']) + + # Substitute variables. + version_str = self._replace_all(self._parts['version'], subs) + + file_strs = {} + for file in version['files']: + subs['last_modified'] = self._get_date(file['last_modified']) + subs['name'] = file['name'] + subs['sha1'] = file['sha1'] + subs['size'] = self._get_file_size(file['size']) + subs['type'] = file['type'] + subs['type_name'] = self._get_type_name(file['type']) + subs['file_url'] = self._get_file_url(platform, version['cef_version'], file) + subs['sha1_url'] = self._get_sha1_url(platform, version['cef_version'], file) + subs['tooltip_text'] = self._get_tooltip_text(platform, version['cef_version'], file) + + # Substitute variables. + file_str = self._replace_all(self._parts['file'], subs) + file_strs[file['type']] = file_str + + if len(file_strs) > 0: + # Always output file types in the same order. + file_out = '' + type_order = ['standard', 'minimal', 'client', 'debug_symbols', 'release_symbols'] + for type in type_order: + if type in file_strs: + file_out = file_out + file_strs[type] + + # Insert files. + version_str = self._replace(version_str, self._section_key('file'), file_out) + version_strs.append(version_str) + + if len(version_strs) > 0: + # Insert versions. + platform_str = self._replace(platform_str, self._section_key('version'), "".join(version_strs)) + platform_strs.append(platform_str) + platform_link_strs.append(platform_link_str) + + if len(platform_strs) > 0: + # Insert platforms. + root_str = self._replace(root_str, self._section_key('platform_link'), "".join(platform_link_strs)) + root_str = self._replace(root_str, self._section_key('platform'), "".join(platform_strs)) + + return root_str + + +# Program entry point. +if __name__ == '__main__': + # Verify command-line arguments. + if len(sys.argv) < 4: + sys.stderr.write('Usage: %s ' % sys.argv[0]) + sys.exit() + + json_file_in = sys.argv[1] + html_file_in = sys.argv[2] + html_file_out = sys.argv[3] + + # Create the HTML builder and load the HTML template. + print '--> Reading %s' % html_file_in + html_builder = cef_html_builder() + with open(html_file_in, 'r') as f: + html_builder.load(f.read()) + + # Create the JSON builder and load the JSON file. + print '--> Reading %s' % json_file_in + json_builder = cef_json_builder(silent=False) + with open(json_file_in, 'r') as f: + json_builder.load(f.read()) + + # Write the HTML output file. + print '--> Writing %s' % html_file_out + with open(html_file_out, 'w') as f: + f.write(html_builder.generate(json_builder)) diff --git a/tools/cefbuilds/cef_json_builder.py b/tools/cefbuilds/cef_json_builder.py new file mode 100644 index 000000000..ef9791b6c --- /dev/null +++ b/tools/cefbuilds/cef_json_builder.py @@ -0,0 +1,434 @@ +# Copyright (c) 2016 The Chromium Embedded Framework Authors. All rights +# reserved. Use of this source code is governed by a BSD-style license that +# can be found in the LICENSE file. + +import datetime +import json +import os +import re +import urllib + +# Class used to build the cefbuilds JSON file. See cef_json_builder_example.py +# for example usage. See cef_json_builder_test.py for unit tests. +# +# Example JSON structure: +# { +# "linux32": { +# "versions": [ +# { +# "cef_version": "3.2704.1414.g185cd6c", +# "chromium_version": "51.0.2704.47" +# "files": [ +# { +# "last_modified": "2016-05-18T22:42:14.066Z" +# "name": "cef_binary_3.2704.1414.g185cd6c_linux32.tar.bz2", +# "sha1": "47c5cfea43912a1d1771f343de35b205f388415f" +# "size": "48549450", +# "type": "standard", +# }, ... +# ], +# }, ... +# ] +# }, ... +# } +# +# Notes: +# - "files" in a given version will be sorted from newest to oldest based on the +# "last_modified" value. +# - "versions" in a given platform will be sorted from newest to oldest based on +# the "last_modified" value of the first (newest) "file" sub-value. +# - There will be at most one record at the "files" level for each "type". + +# This date format intentionally matches the format used in Artifactory +# directory listings. +_CEF_DATE_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ" + +def parse_date(date): + return datetime.datetime.strptime(date, _CEF_DATE_FORMAT) + +def format_date(date): + return date.strftime(_CEF_DATE_FORMAT) + +# Helpers to format datetime values on JSON read/write. +def cef_from_json(json_object): + if 'last_modified' in json_object: + json_object['last_modified'] = parse_date(json_object['last_modified']) + return json_object + +class cef_json_encoder(json.JSONEncoder): + def default(self, o): + if isinstance(o, datetime.datetime): + return format_date(o) + return o + + +class cef_json_builder: + """ Class used to build the cefbuilds JSON file. """ + + def __init__(self, prettyprint=False, silent=True): + """ Create a new cef_json_builder object. """ + self._prettyprint = prettyprint + self._silent = silent + self._fatalerrors = False + self.clear() + + @staticmethod + def get_platforms(): + """ Returns the list of supported platforms. """ + return ('linux32', 'linux64', 'macosx64', 'windows32', 'windows64') + + @staticmethod + def get_distrib_types(): + """ Returns the list of supported distribution types. """ + return ('standard', 'minimal', 'client', 'release_symbols', 'debug_symbols') + + @staticmethod + def is_valid_version(version): + """ Returns true if the specified CEF version is fully qualified and valid. """ + return bool(re.compile('^3.[0-9]{4,5}.[0-9]{4,5}.g[0-9a-f]{7}$').match(version)) + + @staticmethod + def is_valid_chromium_version(version): + """ Returns true if the specified Chromium version is fully qualified and valid. """ + if version is None: + return False + return version == 'master' or \ + re.compile('^[1-9]{1}[0-9]{1}\.0\.[0-9]{4,5}.[0-9]{1,3}$').match(version) + + @staticmethod + def get_file_name(version, platform, type): + """ Returns the expected distribution file name excluding extension based on + the input parameters. """ + if type != 'standard': + type_str = '_' + type + else: + type_str = '' + return 'cef_binary_%s_%s%s' % (version, platform, type_str) + + def clear(self): + """ Clear the contents of this object. """ + self._data = {} + for platform in self.get_platforms(): + self._data[platform] = {'versions': []} + self._versions = {} + self._queryct = 0 + + def __repr__(self): + # Return a string representation of this object. + self._sort_versions() + if self._prettyprint: + return json.dumps(self._data, cls=cef_json_encoder, sort_keys=True, + indent=2, separators=(',', ': ')) + else: + return json.dumps(self._data, cls=cef_json_encoder, sort_keys=True) + + def _print(self, msg): + if self._fatalerrors: + raise Exception(msg) + if not self._silent: + print msg + + def get_query_count(self): + """ Returns the number of queries sent while building. """ + return self._queryct + + def set_chromium_version(self, cef_version, chromium_version=None): + """ Set the matching Chromium version. If the specified Chromium version is + invalid then it will be queried remotely. """ + if not self.is_valid_version(cef_version): + raise Exception('Invalid CEF version: %s' % cef_version) + + if not self.is_valid_chromium_version(chromium_version): + if cef_version in self._versions: + # Keep the Chromium version that we already know about. + return self._versions[cef_version] + + # Try to identify the Chromium version. + chromium_version = 'master' + git_hash = cef_version[-7:] + query_url = 'https://bitbucket.org/chromiumembedded/cef/raw/%s/CHROMIUM_BUILD_COMPATIBILITY.txt' % git_hash + self._queryct = self._queryct + 1 + if not self._silent: + print 'Reading %s' % query_url + + try: + # Read the remote URL contents. + handle = urllib.urlopen(query_url) + compat_value = handle.read().strip() + handle.close() + + # Parse the contents. + config = eval(compat_value, {'__builtins__': None}, None) + if not 'chromium_checkout' in config: + raise Exception('Unexpected contents') + + val = config['chromium_checkout'] + if val.find('refs/tags/') == 0: + chromium_version = val[10:] + except Exception, e: + print 'Failed to read Chromium version information' + raise + + if not self.is_valid_chromium_version(chromium_version): + raise Exception('Invalid Chromium version: %s' % chromium_version) + + self._versions[cef_version] = chromium_version + return chromium_version + + def get_chromium_version(self, cef_version): + """ Return the matching Chromium version. If not currently known it will + be queried remotely. """ + if cef_version in self._versions: + return self._versions[cef_version] + # Identify the Chromium version. + return self.set_chromium_version(cef_version) + + def has_chromium_version(self, cef_version): + """ Return True if a matching Chromium version is known. """ + return cef_version in self._versions + + def load(self, json_string, fatalerrors=True): + """ Load new JSON into this object. Any existing contents will be cleared. + If |fatalerrors| is True then any errors while loading the JSON file + will cause an Exception to be thrown. Otherwise, malformed entries will + will be discarded. Unrecognized keys will always be discarded silently. + """ + self.clear() + + self._fatalerrors = fatalerrors + + new_data = json.JSONDecoder(object_hook = cef_from_json).decode(json_string) + + # Validate the new data's structure. + for platform in self._data.keys(): + if not platform in new_data: + self._print('load: Platform %s not found' % platform) + continue + if not 'versions' in new_data[platform]: + self._print('load: Missing platform key(s) for %s' % platform) + continue + + valid_versions = [] + for version in new_data[platform]['versions']: + if not 'cef_version' in version or \ + not 'chromium_version' in version or \ + not 'files' in version: + self._print('load: Missing version key(s) for %s' % platform) + continue + + valid_files = [] + found_types = [] + for file in version['files']: + if not 'type' in file or \ + not 'name' in file or \ + not 'size' in file or \ + not 'last_modified' in file or \ + not 'sha1' in file: + self._print('load: Missing file key(s) for %s %s' % (platform, version['cef_version'])) + continue + (expected_platform, expected_version, expected_type) = self._parse_name(file['name']) + if expected_platform != platform or \ + expected_version != version['cef_version'] or \ + expected_type != file['type']: + self._print('load: File name/attribute mismatch for %s %s %s' % + (platform, version['cef_version'], file['name'])) + continue + self._validate_args(platform, version['cef_version'], file['type'], + file['size'], file['last_modified'], file['sha1']) + if file['type'] in found_types: + self._print('load: Duplicate %s type for %s %s' % (file['type'], platform, version['cef_version'])) + continue + found_types.append(file['type']) + valid_files.append({ + 'type': file['type'], + 'name': file['name'], + 'size': file['size'], + 'last_modified': file['last_modified'], + 'sha1': file['sha1'], + }) + + if len(valid_files) > 0: + valid_versions.append({ + 'cef_version': version['cef_version'], + 'chromium_version': self.set_chromium_version(version['cef_version'], version['chromium_version']), + 'files': self._sort_files(valid_files) + }) + + if len(valid_versions) > 0: + self._data[platform]['versions'] = valid_versions + + self._fatalerrors = False + + def _sort_versions(self): + # Sort version records by first (newest) file last_modified value. + for platform in self._data.keys(): + for i in range(0, len(self._data[platform]['versions'])): + self._data[platform]['versions'] = \ + sorted(self._data[platform]['versions'], + key=lambda k: k['files'][0]['last_modified'], + reverse=True) + + @staticmethod + def _sort_files(files): + # Sort file records by last_modified. + return sorted(files, key=lambda k: k['last_modified'], reverse=True) + + @staticmethod + def _parse_name(name): + # Remove file extension. + name_no_ext = os.path.splitext(name)[0] + if name_no_ext[-4:] == '.tar': + name_no_ext = name_no_ext[:-4] + name_parts = name_no_ext.split('_') + if len(name_parts) < 4 or name_parts[0] != 'cef' or name_parts[1] != 'binary': + raise Exception('Invalid filename: %s' % name) + + # Remove 'cef' and 'binary'. + del name_parts[0] + del name_parts[0] + + type = None + + # Might be '__[debug|release]_symbols'. + if name_parts[-1] == 'symbols': + del name_parts[-1] + if name_parts[-1] == 'debug' or name_parts[-1] == 'release': + type = name_parts[-1] + '_symbols' + del name_parts[-1] + + # Might be '__minimal'. + if name_parts[-1] == 'minimal': + type = 'minimal' + del name_parts[-1] + + # Might be '__client'. + if name_parts[-1] == 'client': + type = 'client' + del name_parts[-1] + + # Remainder must be '_'. + if len(name_parts) != 2: + raise Exception('Invalid filename: %s' % name) + + if type is None: + type = 'standard' + + version = name_parts[0] + platform = name_parts[1] + + return [platform, version, type] + + @staticmethod + def _validate_args(platform, version, type, size, last_modified, sha1): + # Validate input arguments. + if not platform in cef_json_builder.get_platforms(): + raise Exception('Unsupported platform: %s' % platform) + + if not cef_json_builder.is_valid_version(version): + raise Exception('Invalid version: %s' % version) + + if not type in cef_json_builder.get_distrib_types(): + raise Exception('Unsupported distribution type: %s' % type) + + if int(size) <= 0: + raise Exception('Invalid size: %s' % size) + + if not isinstance(last_modified, datetime.datetime): + # datetime will throw a ValueException if it doesn't parse. + parse_date(last_modified) + + if not re.compile('^[0-9a-f]{40}$').match(sha1): + raise Exception('Invalid sha1: %s' % sha1) + + def add_file(self, name, size, last_modified, sha1): + """ Add a file record with the specified attributes. Returns True if the + file is added or False if a file with the same |name| and |sha1| + already exists. """ + # Parse the file name. + (platform, version, type) = self._parse_name(name) + + if not isinstance(size, (int, long)): + size = int(size) + if not isinstance(last_modified, datetime.datetime): + last_modified = parse_date(last_modified) + + # Validate arguments. + self._validate_args(platform, version, type, size, last_modified, sha1) + + # Find the existing version record. + version_idx = -1; + for i in range(0, len(self._data[platform]['versions'])): + if self._data[platform]['versions'][i]['cef_version'] == version: + # Check the version record. + self._print('add_file: Check %s %s' % (platform, version)) + version_idx = i + break + + if version_idx == -1: + # Add a new version record. + self._print('add_file: Add %s %s' % (platform, version)) + self._data[platform]['versions'].append({ + 'cef_version': version, + 'chromium_version': self.get_chromium_version(version), + 'files': [] + }) + version_idx = len(self._data[platform]['versions']) - 1 + + # Find the existing file record with matching type. + file_changed = True + for i in range(0, len(self._data[platform]['versions'][version_idx]['files'])): + if self._data[platform]['versions'][version_idx]['files'][i]['type'] == type: + existing_sha1 = self._data[platform]['versions'][version_idx]['files'][i]['sha1'] + if existing_sha1 != sha1: + # Remove the existing file record. + self._print(' Remove %s %s' % (name, existing_sha1)) + del self._data[platform]['versions'][version_idx]['files'][i] + else: + file_changed = False + break + + if file_changed: + # Add a new file record. + self._print(' Add %s %s' % (name, sha1)) + self._data[platform]['versions'][version_idx]['files'].append({ + 'type': type, + 'name': name, + 'size': size, + 'last_modified': last_modified, + 'sha1': sha1 + }) + + # Sort file records by last_modified. + # This is necessary for _sort_versions() to function correctly. + self._data[platform]['versions'][version_idx]['files'] = \ + self._sort_files(self._data[platform]['versions'][version_idx]['files']) + + return file_changed + + def get_files(self, platform=None, version=None, type=None): + """ Return the files that match the input parameters. + All parameters are optional. Version will do partial matching. """ + results = [] + + if platform is None: + platforms = self._data.keys() + else: + platforms = [platform] + + for platform in platforms: + for version_obj in self._data[platform]['versions']: + if version is None or version_obj['cef_version'].find(version) == 0: + for file_obj in version_obj['files']: + if type is None or type == file_obj['type']: + result_obj = file_obj; + # Add additional metadata. + result_obj['platform'] = platform + result_obj['cef_version'] = version_obj['cef_version'] + result_obj['chromium_version'] = version_obj['chromium_version'] + results.append(result_obj) + + return results + + def get_versions(self, platform): + """ Return all versions for the specified |platform|. """ + return self._data[platform]['versions'] diff --git a/tools/cefbuilds/cef_json_builder_example.py b/tools/cefbuilds/cef_json_builder_example.py new file mode 100644 index 000000000..0b4e63d9b --- /dev/null +++ b/tools/cefbuilds/cef_json_builder_example.py @@ -0,0 +1,144 @@ +# Copyright (c) 2016 The Chromium Embedded Framework Authors. All rights +# reserved. Use of this source code is governed by a BSD-style license that +# can be found in the LICENSE file. + +# This example utility uses the cef_json_builder class to create and update an +# index.json file in the same directory as this file. Example usage: +# +# Add files for macosx64 platform at the specified version: +# > python cef_json_builder_example.py add macosx64 3.2704.1416.g185cd6c 51.0.2704.47 +# +# Add files for all platforms at the specified version: +# > python cef_json_builder_example.py add all 3.2704.1416.g185cd6c 51.0.2704.47 +# +# See cef_json_builder.get_platforms() for the list of supported platforms. +# +# After creating an index.json file you can use the cef_html_builder.py tool to +# create an HTML file. + +from cef_json_builder import cef_json_builder +import datetime +import os +import random +import string +import sys + +# Create a fake sha1 checksum value. +def make_fake_sha1(): + return ''.join(random.SystemRandom().choice('abcdef' + string.digits) for _ in range(40)) + +# Create a fake file size value. +def make_fake_size(): + return random.randint(30000000, 60000000) + +# Create fake file info based on |platform| and |version|. +def make_fake_file_info(platform, version, type): + return { + 'name': cef_json_builder.get_file_name(version, platform, type) + '.tar.gz', + 'size': make_fake_size(), + 'lastModified': datetime.datetime.now(), + 'sha1': make_fake_sha1() + } + +# Returns a list of fake files based on |platform| and |version|. +def create_fake_files(platform, version): + files = [] + + # All platforms create standard and minimal distributions. + files.append(make_fake_file_info(platform, version, 'standard')) + files.append(make_fake_file_info(platform, version, 'minimal')) + + # Non-Linux platforms create client distributions. + if platform.find('linux') == -1: + files.append(make_fake_file_info(platform, version, 'client')) + + # Windows platforms create debug symbols. + if platform.find('windows') == 0: + files.append(make_fake_file_info(platform, version, 'debug_symbols')) + + # Windows and OS X platforms create release symbols. + if platform.find('windows') == 0 or platform.find('macosx') == 0: + files.append(make_fake_file_info(platform, version, 'release_symbols')) + + return files + + +# Program entry point. +if __name__ == '__main__': + # Verify command-line arguments. + if len(sys.argv) < 5 or sys.argv[1] != 'add': + sys.stderr.write('Usage: %s add ' % sys.argv[0]) + sys.exit() + + # Requested platform. + if sys.argv[2] == 'all': + platforms = cef_json_builder.get_platforms() + elif sys.argv[2] in cef_json_builder.get_platforms(): + platforms = [sys.argv[2]] + else: + sys.stderr.write('Invalid platform: %s' % platform) + sys.exit() + + # Requested CEF version. + cef_version = sys.argv[3] + if not cef_json_builder.is_valid_version(cef_version): + sys.stderr.write('Invalid CEF version: %s' % cef_version) + sys.exit() + + # Requested Chromium version. + chromium_version = sys.argv[4] + if not cef_json_builder.is_valid_chromium_version(chromium_version): + sys.stderr.write('Invalid Chromium version: %s' % chromium_version) + sys.exit() + + # Write the JSON file in the same directory as this file. + current_dir = os.path.dirname(__file__) + json_file = os.path.join(current_dir, 'index.json') + + # Create the builder object. Enable pretty printing and extra output for + # example purposes. + builder = cef_json_builder(prettyprint=True, silent=False) + + # Map the CEF version to the Chromium version to avoid a remote query. + builder.set_chromium_version(cef_version, chromium_version) + + # Load the existing JSON file, if any. Ignore format errors for example + # purposes. + if os.path.exists(json_file): + print '--> Reading index.json' + with open(json_file, 'r') as f: + builder.load(f.read(), fatalerrors=False) + + # Create fake file info based on |platform| and |version|. A real + # implementation should retrieve the list of files from an external source + # like a Web or filesystem directory listing. If using Artifactory, for + # example, then "size", "lastModified" and "sha1" attributes would be included + # in the directory listing metadata. + # For this example we: + # - Always use now() as the last modified date. Consequently newly added files + # will always be listed at the top of the JSON platform versions list. + # - Always create a new (fake) sha1 checksum value for each file. Consequently + # duplicate calls for the same |platform| + |version| will always replace + # the existing entries. In real implementations the sha1 may be the same + # across calls if the file contents have not changed, in which case + # cef_json_builder.add_file() will return False and upload of the file + # should be skipped. + new_files = [] + for platform in platforms: + new_files.extend(create_fake_files(platform, cef_version)) + + # Add new files to the builder. + changed_files = [] + for file in new_files: + if builder.add_file(file['name'], file['size'], file['lastModified'], file['sha1']): + changed_files.append(file) + + if len(changed_files) > 0: + # Write the updated JSON file. + print '--> Writing index.json' + with open(json_file, 'w') as f: + f.write(str(builder)) + + # A real implementation would now upload the changed files. + for file in changed_files: + print '--> Upload file %s' % file['name'] diff --git a/tools/cefbuilds/cef_json_builder_test.py b/tools/cefbuilds/cef_json_builder_test.py new file mode 100644 index 000000000..a0b0070b6 --- /dev/null +++ b/tools/cefbuilds/cef_json_builder_test.py @@ -0,0 +1,318 @@ +# Copyright (c) 2016 The Chromium Embedded Framework Authors. All rights +# reserved. Use of this source code is governed by a BSD-style license that +# can be found in the LICENSE file. + +from cef_json_builder import cef_json_builder +import datetime +import unittest + +class TestCefJSONBuilder(unittest.TestCase): + + # Write builder contents to string and then read in. + def _verify_write_read(self, builder): + output = str(builder) + builder2 = cef_json_builder() + builder2.load(output) + self.assertEqual(output, str(builder2)) + + # Add a file record for testing purposes. + def _add_test_file(self, builder, platform='linux32', version='3.2704.1414.g185cd6c', + type='standard', attrib_idx=0, shouldfail=False): + name = cef_json_builder.get_file_name(version, platform, type) + '.tar.gz' + + # Some random attribute information. sha1 must be different to trigger replacement. + attribs = [ + { + 'date_str': '2016-05-18T22:42:15.487Z', + 'date_val': datetime.datetime(2016, 5, 18, 22, 42, 15, 487000), + 'sha1': '2d48ee05ea6385c8fe80879c98c5dd505ad4b100', + 'size': 48395610 + }, + { + 'date_str': '2016-05-14T22:42:15.487Z', + 'date_val': datetime.datetime(2016, 5, 14, 22, 42, 15, 487000), + 'sha1': '2d48ee05ea6385c8fe80879c98c5dd505ad4b200', + 'size': 48395620 + } + ] + + # Populate the Chromium version to avoid queries. + chromium_version = '49.0.2705.50' + self.assertEqual(chromium_version, builder.set_chromium_version(version, chromium_version)) + self.assertEqual(0, builder.get_query_count()) + + result = builder.add_file(name, + attribs[attrib_idx]['size'], + attribs[attrib_idx]['date_str'], + attribs[attrib_idx]['sha1']) + # Failure should be expected when adding the same file multiple times with the same sha1. + self.assertEqual(not shouldfail, result) + + # Return the result expected from get_files(). + return { + 'chromium_version': chromium_version, + 'sha1': attribs[attrib_idx]['sha1'], + 'name': name, + 'platform': platform, + 'last_modified': attribs[attrib_idx]['date_val'], + 'cef_version': version, + 'type': type, + 'size': attribs[attrib_idx]['size'] + } + + # Test with no file contents. + def test_empty(self): + builder = cef_json_builder() + + self._verify_write_read(builder) + + files = builder.get_files() + self.assertEqual(len(files), 0) + + # Test add/get of a single file with the specified type. + def _test_add_file(self, type): + builder = cef_json_builder() + + expected = self._add_test_file(builder, type=type) + self._verify_write_read(builder) + + files = builder.get_files() + self.assertEqual(len(files), 1) + self.assertEqual(expected, files[0]) + + # Test add/get of a standard type file. + def test_add_standard_file(self): + self._test_add_file('standard') + + # Test add/get of a minimal type file. + def test_add_minimal_file(self): + self._test_add_file('minimal') + + # Test add/get of a client type file. + def test_add_client_file(self): + self._test_add_file('client') + + # Test add/get of a debug_symbols type file. + def test_add_debug_symbols_file(self): + self._test_add_file('debug_symbols') + + # Test add/get of a release_symbols type file. + def test_add_release_symbols_file(self): + self._test_add_file('release_symbols') + + # Test get_files() behavior with a single file. + def test_get_files_single(self): + builder = cef_json_builder() + + # Specify all values just in case the defaults change. + expected = self._add_test_file(builder, + platform='linux32', version='3.2704.1414.g185cd6c', type='standard') + + # No filter. + files = builder.get_files() + self.assertEqual(len(files), 1) + self.assertEqual(expected, files[0]) + + # Platform filter. + files = builder.get_files(platform='linux32') + self.assertEqual(len(files), 1) + self.assertEqual(expected, files[0]) + files = builder.get_files(platform='linux64') + self.assertEqual(len(files), 0) + + # Version filter exact. + files = builder.get_files(version='3.2704.1414.g185cd6c') + self.assertEqual(len(files), 1) + self.assertEqual(expected, files[0]) + + # Version filter partial. + files = builder.get_files(version='3.2704') + self.assertEqual(len(files), 1) + self.assertEqual(expected, files[0]) + files = builder.get_files(version='3.2623') + self.assertEqual(len(files), 0) + + # Type filter. + files = builder.get_files(type='standard') + self.assertEqual(len(files), 1) + self.assertEqual(expected, files[0]) + files = builder.get_files(type='client') + self.assertEqual(len(files), 0) + + # All filters. + files = builder.get_files(platform='linux32', version='3.2704', type='standard') + self.assertEqual(len(files), 1) + self.assertEqual(expected, files[0]) + files = builder.get_files(platform='linux32', version='3.2704', type='minimal') + self.assertEqual(len(files), 0) + files = builder.get_files(platform='linux32', version='3.2623', type='standard') + self.assertEqual(len(files), 0) + files = builder.get_files(platform='linux64', version='3.2704', type='standard') + self.assertEqual(len(files), 0) + + # Test add/get of multiple files. + def test_add_multiple_files(self): + builder = cef_json_builder() + + expected = [] + + platforms = cef_json_builder.get_platforms() + versions = ['3.2704.1414.g185cd6c', '3.2704.1400.gde36543'] + types = cef_json_builder.get_distrib_types() + for platform in platforms: + for version in versions: + for type in types: + expected.append(self._add_test_file(builder, platform=platform, type=type, version=version)) + + self._verify_write_read(builder) + + # No filter. + files = builder.get_files() + self.assertEqual(len(files), len(platforms) * len(versions) * len(types)) + + # Version filter. + files = builder.get_files(version=version) + self.assertEqual(len(files), len(platforms) * len(types)) + + # Type filter. + files = builder.get_files(type='client') + self.assertEqual(len(files), len(platforms) * len(versions)) + + # Platform filter. + files = builder.get_files(platform='windows32') + self.assertEqual(len(files), len(types) * len(versions)) + + # All filters. + idx = 0 + for platform in platforms: + for version in versions: + for type in types: + files = builder.get_files(platform=platform, type=type, version=version) + self.assertEqual(len(files), 1) + self.assertEqual(expected[idx], files[0]) + idx = idx + 1 + + # Test add/get/replace of multiple files. + def test_replace_all_files(self): + builder = cef_json_builder() + + version = '3.2704.1414.g185cd6c' + platforms = ['linux32', 'linux64'] + types = ['standard', 'minimal'] + + # Initial file versions. + for platform in platforms: + for type in types: + self._add_test_file(builder, platform=platform, type=type, version=version) + + # No filter. + files = builder.get_files() + self.assertEqual(len(files), len(platforms) * len(types)) + + expected = [] + + # Replace all file versions (due to new sha1). + for platform in platforms: + for type in types: + expected.append(self._add_test_file(builder, + platform=platform, type=type, version=version, attrib_idx=1)) + + # No filter. + files = builder.get_files() + self.assertEqual(len(files), len(platforms) * len(types)) + + # All filters. + idx = 0 + for platform in platforms: + for type in types: + files = builder.get_files(platform=platform, type=type, version=version) + self.assertEqual(len(files), 1) + self.assertEqual(expected[idx], files[0]) + idx = idx + 1 + + # Test add/get/no replace of multiple files. + def test_replace_no_files(self): + builder = cef_json_builder() + + version = '3.2704.1414.g185cd6c' + platforms = ['linux32', 'linux64'] + types = ['standard', 'minimal'] + + # Initial file versions. + for platform in platforms: + for type in types: + self._add_test_file(builder, platform=platform, type=type, version=version) + + # No filter. + files = builder.get_files() + self.assertEqual(len(files), len(platforms) * len(types)) + + expected = [] + + # Replace no file versions (due to same sha1). + for platform in platforms: + for type in types: + expected.append(self._add_test_file(builder, + platform=platform, type=type, version=version, shouldfail=True)) + + # No filter. + files = builder.get_files() + self.assertEqual(len(files), len(platforms) * len(types)) + + # All filters. + idx = 0 + for platform in platforms: + for type in types: + files = builder.get_files(platform=platform, type=type, version=version) + self.assertEqual(len(files), 1) + self.assertEqual(expected[idx], files[0]) + idx = idx + 1 + + # Test Chromium version. + def test_chromium_version(self): + builder = cef_json_builder() + + self.assertTrue(builder.is_valid_chromium_version('master')) + self.assertTrue(builder.is_valid_chromium_version('49.0.2704.0')) + self.assertTrue(builder.is_valid_chromium_version('49.0.2704.50')) + self.assertTrue(builder.is_valid_chromium_version('49.0.2704.100')) + + self.assertFalse(builder.is_valid_chromium_version(None)) + self.assertFalse(builder.is_valid_chromium_version('09.0.2704.50')) + self.assertFalse(builder.is_valid_chromium_version('00.0.0000.00')) + self.assertFalse(builder.is_valid_chromium_version('foobar')) + + # The Git hashes must exist but the rest of the CEF version can be fake. + versions = ( + ('3.2704.1414.g185cd6c', '51.0.2704.47'), + ('3.2623.9999.gb90a3be', '49.0.2623.110'), + ('3.2623.9999.g2a6491b', '49.0.2623.87'), + ('3.9999.9999.gab2636b', 'master'), + ) + + # Test with no query. + for (cef, chromium) in versions: + self.assertFalse(builder.has_chromium_version(cef)) + # Valid version specified, so no query sent. + self.assertEqual(chromium, builder.set_chromium_version(cef, chromium)) + self.assertEqual(chromium, builder.get_chromium_version(cef)) + self.assertTrue(builder.has_chromium_version(cef)) + # Ignore attempts to set invalid version after setting valid version. + self.assertEqual(chromium, builder.set_chromium_version(cef, None)) + + self.assertEqual(0, builder.get_query_count()) + builder.clear() + + # Test with query. + for (cef, chromium) in versions: + self.assertFalse(builder.has_chromium_version(cef)) + # No version known, so query sent. + self.assertEqual(chromium, builder.get_chromium_version(cef)) + self.assertTrue(builder.has_chromium_version(cef)) + + self.assertEqual(len(versions), builder.get_query_count()) + + +# Program entry point. +if __name__ == '__main__': + unittest.main() diff --git a/tools/cefbuilds/index.html.in b/tools/cefbuilds/index.html.in new file mode 100644 index 000000000..9b0adec8c --- /dev/null +++ b/tools/cefbuilds/index.html.in @@ -0,0 +1,178 @@ + + + + CEF Automated Builds + + + + + + + + + + + + + + + +
+ Chromium Embedded Framework (CEF) Automated Builds +
[ + Project Page | + Support Forum + ]		 + $branding$ +
+
+ Builds: + + ▶ $platform_name$ + +

+ +
+

+ + + $platform_name$ Builds + + + + + + + + + +
+ + + +
+ + + + + + + + +
$last_modified$ - CEF $cef_version$ / Chromium $chromium_version$CEF source | Chromium source
+ + + + + + + + + +
▶
$type_name$$tooltip_text$
$name$$size$sha1
+
+
+
+ +

Copyright © $year$ The Chromium Embedded Framework Authors. All rights reserved.

+ +