From 1e880c56ed6efce9cd45024ac4e6701fead2d5cf Mon Sep 17 00:00:00 2001 From: Marshall Greenblatt Date: Wed, 10 Jul 2024 16:11:27 -0400 Subject: [PATCH] distrib: Add new tools distribution for mksnapshot (see #3734) --- tools/automate/automate-git.py | 33 ++++-- tools/distrib/README.tools.txt | 58 ++++++++++ tools/distrib/tools/run_mksnapshot.bat | 66 +++++++++++ tools/distrib/tools/run_mksnapshot.sh | 68 ++++++++++++ tools/make_distrib.py | 145 ++++++++++++++++++++++++- 5 files changed, 359 insertions(+), 11 deletions(-) create mode 100644 tools/distrib/README.tools.txt create mode 100644 tools/distrib/tools/run_mksnapshot.bat create mode 100755 tools/distrib/tools/run_mksnapshot.sh diff --git a/tools/automate/automate-git.py b/tools/automate/automate-git.py index f1ec60b17..cb0ae3bde 100644 --- a/tools/automate/automate-git.py +++ b/tools/automate/automate-git.py @@ -461,6 +461,7 @@ def check_pattern_matches(output_file=None): # Don't continue when we know the build will be wrong. sys.exit(1) + def invalid_options_combination(a, b): print("Invalid combination of options: '%s' and '%s'" % (a, b)) parser.print_help(sys.stderr) @@ -767,6 +768,18 @@ parser.add_option( dest='sandboxdistribonly', default=False, help='Create a cef_sandbox static library distribution only.') +parser.add_option( + '--tools-distrib', + action='store_true', + dest='toolsdistrib', + default=False, + help='Create a tools distribution.') +parser.add_option( + '--tools-distrib-only', + action='store_true', + dest='toolsdistribonly', + default=False, + help='Create a tools distribution only.') parser.add_option( '--no-distrib-docs', action='store_true', @@ -814,22 +827,22 @@ if options.runtests: options.buildtests = True if (options.nochromiumupdate and options.forceupdate): - invalid_options_combination('--no-chromium-update', '--force-update') + invalid_options_combination('--no-chromium-update', '--force-update') if (options.nocefupdate and options.forceupdate): - invalid_options_combination('--no-cef-update', '--force-update') + invalid_options_combination('--no-cef-update', '--force-update') if (options.nobuild and options.forcebuild): - invalid_options_combination('--no-build', '--force-build') + invalid_options_combination('--no-build', '--force-build') if (options.nodistrib and options.forcedistrib): - invalid_options_combination('--no-distrib', '--force-distrib') + invalid_options_combination('--no-distrib', '--force-distrib') if (options.forceclean and options.fastupdate): - invalid_options_combination('--force-clean', '--fast-update') + invalid_options_combination('--force-clean', '--fast-update') if (options.forcecleandeps and options.fastupdate): - invalid_options_combination('--force-clean-deps', '--fast-update') + invalid_options_combination('--force-clean-deps', '--fast-update') if (options.noreleasebuild and \ (options.minimaldistrib or options.minimaldistribonly or \ options.clientdistrib or options.clientdistribonly)) or \ - (options.minimaldistribonly + options.clientdistribonly + options.sandboxdistribonly > 1): + (options.minimaldistribonly + options.clientdistribonly + options.sandboxdistribonly + options.toolsdistribonly > 1): print('Invalid combination of options.') parser.print_help(sys.stderr) sys.exit(1) @@ -1450,6 +1463,8 @@ if not options.nodistrib and (chromium_checkout_changed or \ distrib_types.append('client') elif options.sandboxdistribonly: distrib_types.append('sandbox') + elif options.toolsdistribonly: + distrib_types.append('tools') else: distrib_types.append('standard') if options.minimaldistrib: @@ -1458,6 +1473,8 @@ if not options.nodistrib and (chromium_checkout_changed or \ distrib_types.append('client') if options.sandboxdistrib: distrib_types.append('sandbox') + if options.toolsdistrib: + distrib_types.append('tools') cef_tools_dir = os.path.join(cef_src_dir, 'tools') @@ -1482,6 +1499,8 @@ if not options.nodistrib and (chromium_checkout_changed or \ path += ' --client' elif type == 'sandbox': path += ' --sandbox' + elif type == 'tools': + path += ' --tools' if first_type: if options.nodistribdocs: diff --git a/tools/distrib/README.tools.txt b/tools/distrib/README.tools.txt new file mode 100644 index 000000000..a08254906 --- /dev/null +++ b/tools/distrib/README.tools.txt @@ -0,0 +1,58 @@ +CONTENTS +-------- + +Debug Contains the Debug build of tools. + +Release Contains the Release build of tools. + + +IMPORTANT NOTE +-------------- + +CEF/Chromium builds are created using the following host architectures: + +- Linux: x86-64 (Intel/AMD) +- Windows: x86-64 (Intel/AMD) +- MacOS: ARM64 (Apple Silicon) + +Binaries in this tools package must be run on the supported host OS/architecture +even in cases where the output targets a different architecture. + +For example, files targeting a MacOS 64-bit (Intel) application must be created +on a MacOS ARM64 (Apple Silicon) host system using the MacOS 64-bit (Intel) +tools distribution. + + +USAGE +----- + +Start with the required host system and the tools distribution that matches your +application's target OS/architecture and CEF version. + +Custom V8 Snapshots + +Custom startup snapshots [https://v8.dev/blog/custom-startup-snapshots] can be +used to speed up V8/JavaScript load time in the renderer process. With CEF this +works as follows: + +1. Generate a single JavaScript file that contains custom startup data. For + example, using https://github.com/atom/electron-link. + +2. Execute the `run_mksnapshot` script to create a `v8_context_snapshot.bin` + file containing the custom data in addition to the default V8 data. + + Example: + % run_mksnapshot Release /path/to/snapshot.js + + Note that bin file names include an architecture component on MacOS + (e.g. `v8_context_snapshot.[arm64|x86_64].bin`) + +3. Replace the existing `v8_context_snapshot.bin` file in the installation + folder or app bundle. + +4. Run the application and verify in DevTools that the custom startup data + exists. For example, electron-link adds a global `snapshotResult` object. + +Please visit the CEF Website for additional usage information. + +https://bitbucket.org/chromiumembedded/cef/ diff --git a/tools/distrib/tools/run_mksnapshot.bat b/tools/distrib/tools/run_mksnapshot.bat new file mode 100644 index 000000000..0edccf336 --- /dev/null +++ b/tools/distrib/tools/run_mksnapshot.bat @@ -0,0 +1,66 @@ +@echo off +:: Copyright (c) 2024 The Chromium Embedded Framework Authors. All rights +:: reserved. Use of this source code is governed by a BSD-style license +:: that can be found in the LICENSE file. + +set RC= + +setlocal + +if not "%1" == "Debug" ( + if not "%1" == "Release" ( + echo Usage: run_mksnapshot.bat [Debug^|Release] path\to\snapshot.js + set ERRORLEVEL=1 + goto end + ) +) + +set SCRIPT_DIR=%~dp0 +set BIN_DIR=%SCRIPT_DIR%%~1 + +if not exist "%BIN_DIR%" ( + echo %BIN_DIR% directory not found + set ERRORLEVEL=1 + goto end +) + +set CMD_FILE=mksnapshot_cmd.txt + +if not exist "%BIN_DIR%\%CMD_FILE%" ( + echo %BIN_DIR%\%CMD_FILE% file not found + set ERRORLEVEL=1 + goto end +) + +cd "%BIN_DIR%" + +:: Read %CMD_FILE% into a local variable. +set /p CMDLINE=<%CMD_FILE% + +:: Generate snapshot_blob.bin. +echo Running mksnapshot... +call mksnapshot %CMDLINE% %2 + +set OUT_FILE=v8_context_snapshot.bin + +:: Generate v8_context_snapshot.bin. +echo Running v8_context_snapshot_generator... +call v8_context_snapshot_generator --output_file=%OUT_FILE% + +if not exist "%BIN_DIR%\%OUT_FILE%" ( + echo Failed + set ERRORLEVEL=1 + goto end +) + +echo Success! Created %BIN_DIR%\%OUT_FILE% + +:end +endlocal & set RC=%ERRORLEVEL% +goto omega + +:returncode +exit /B %RC% + +:omega +call :returncode %RC% diff --git a/tools/distrib/tools/run_mksnapshot.sh b/tools/distrib/tools/run_mksnapshot.sh new file mode 100755 index 000000000..803c19151 --- /dev/null +++ b/tools/distrib/tools/run_mksnapshot.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# Copyright (c) 2024 The Chromium Embedded Framework Authors. All rights +# reserved. Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file. + +if [ "$1" != "Debug" ] && [ "$1" != "Release" ]; then + echo 'Usage: run_mksnapshot.sh [Debug|Release] /path/to/snapshot.js' + exit 1 +fi + +SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" +BIN_DIR=$SCRIPT_DIR/$1 + +if [ ! -d "$BIN_DIR" ]; then + echo "$BIN_DIR directory not found." + exit 1 +fi + +CMD_FILE=mksnapshot_cmd.txt + +if [ ! -f "$BIN_DIR/$CMD_FILE" ]; then + echo "$BIN_DIR/$CMD_FILE file not found." + exit 1 +fi + +pushd "$BIN_DIR" > /dev/null + +# Read $CMD_FILE into an argument array. +IFS=' ' read -r -a args < $CMD_FILE + +if [ "$(uname)" == "Darwin" ]; then + # Execution of tools binaries downloaded on MacOS may be blocked + # by Apple Security settings with a message like " can't + # be opened because Apple cannot check it for malicious software." + # Remove that block here. + xattr -c ./mksnapshot + xattr -c ./v8_context_snapshot_generator +fi + +# Generate snapshot_blob.bin. +echo 'Running mksnapshot...' +./mksnapshot "${args[@]}" "${@:2}" + +# Determine the architecture suffix, if any. +suffix='' +if [ "$(uname)" == "Darwin" ]; then + value='--target_arch=arm64' + if [[ " ${args[*]} " =~ [[:space:]]${value}[[:space:]] ]]; then + suffix='.arm64' + else + suffix='.x86_64' + fi +fi + +OUT_FILE=v8_context_snapshot${suffix}.bin + +# Generate v8_context_snapshot.bin. +echo 'Running v8_context_snapshot_generator...' +./v8_context_snapshot_generator --output_file=$OUT_FILE + +popd > /dev/null + +if [ -f "$BIN_DIR/$OUT_FILE" ]; then + echo "Success! Created $BIN_DIR/$OUT_FILE" +else + echo "Failed" + exit 1 +fi diff --git a/tools/make_distrib.py b/tools/make_distrib.py index 712bda851..bc8f13348 100644 --- a/tools/make_distrib.py +++ b/tools/make_distrib.py @@ -122,7 +122,7 @@ def create_readme(): # format the file data = header_data + '\n\n' + mode_data - if mode != 'sandbox': + if mode != 'sandbox' and mode != 'tools': data += '\n\n' + redistrib_data data += '\n\n' + footer_data data = data.replace('$CEF_URL$', cef_url) @@ -165,6 +165,9 @@ def create_readme(): distrib_type = 'Sandbox' distrib_desc = 'This distribution contains only the cef_sandbox static library. Please see\n' \ 'the LICENSING section of this document for licensing terms and conditions.' + elif mode == 'tools': + distrib_type = 'Tools' + distrib_desc = 'This distribution contains additional tools for building CEF-based applications.' data = data.replace('$DISTRIB_TYPE$', distrib_type) data = data.replace('$DISTRIB_DESC$', distrib_desc) @@ -232,6 +235,128 @@ def transfer_gypi_files(src_dir, gypi_paths, gypi_path_prefix, dst_dir, quiet): copy_file(src, dst, quiet) +def extract_toolchain_cmd(build_dir, + exe_name, + require_toolchain, + require_cmd=True): + """ Extract a toolchain command from the ninja configuration file. """ + toolchain_ninja = os.path.join(build_dir, 'toolchain.ninja') + if not os.path.isfile(toolchain_ninja): + if not require_toolchain: + return None, None + raise Exception('Missing file: %s' % toolchain_ninja) + + data = read_file(toolchain_ninja) + + cmd = None + path = None + + # Looking for a value like: + # command = python3 ../../v8/tools/run.py ./exe_name --arg1 --arg2 + # OR (for cross-compile): + # command = python3 ../../v8/tools/run.py ./clang_arch1_arch2/exe_name --arg1 --arg2 + findstr = '/%s ' % exe_name + start = data.find(findstr) + if start >= 0: + # Extract the command-line arguments. + after_start = start + len(findstr) + end = data.find('\n', after_start) + if end >= after_start: + cmd = data[after_start:end].strip() + print('%s command:' % exe_name, cmd) + if cmd != '' and not re.match(r"^[0-9a-zA-Z\_\- ./=]{1,}$", cmd): + cmd = None + + # Extract the relative file path. + dot = start - 1 + while data[dot].isalnum() or data[dot] == '_': + dot -= 1 + path = data[dot + 1:start] + print('%s path:' % exe_name, path) + if path != '' and not re.match(r"^(win_)?clang_[0-9a-z_]{1,}$", path): + path = None + + if require_cmd and (cmd is None or path is None): + raise Exception('Failed to extract %s command from %s' % (exe_name, + toolchain_ninja)) + + return cmd, path + + +def get_exe_name(exe_name): + return exe_name + ('.exe' if platform == 'windows' else '') + + +def get_script_name(script_name): + return script_name + ('.bat' if platform == 'windows' else '.sh') + + +def transfer_tools_files(script_dir, build_dirs, output_dir): + for build_dir in build_dirs: + is_debug = build_dir.find('Debug') >= 0 + dst_dir_name = 'Debug' if is_debug else 'Release' + dst_dir = os.path.join(output_dir, dst_dir_name) + + # Retrieve the binary path and command-line arguments. + # See issue #3734 for the expected format. + mksnapshot_name = 'mksnapshot' + tool_cmd, tool_dir = extract_toolchain_cmd( + build_dir, mksnapshot_name, require_toolchain=not options.allowpartial) + if tool_cmd is None: + sys.stdout.write("No %s build toolchain for %s.\n" % (dst_dir_name, + mksnapshot_name)) + continue + + if options.allowpartial and not path_exists( + os.path.join(build_dir, tool_dir, get_exe_name(mksnapshot_name))): + sys.stdout.write("No %s build of %s.\n" % (dst_dir_name, mksnapshot_name)) + continue + + # yapf: disable + binaries = [ + {'path': get_exe_name(mksnapshot_name)}, + {'path': get_exe_name('v8_context_snapshot_generator')}, + ] + # yapf: disable + + # Transfer binaries. + copy_files_list(os.path.join(build_dir, tool_dir), dst_dir, binaries) + + # Evaluate command-line arguments and remove relative paths. Copy any input files + # into the distribution. + # - Example input path : ../../v8/tools/builtins-pgo/profiles/x64-rl.profile + # - Example output path: gen/v8/embedded.S + parsed_cmd = [] + for cmd in tool_cmd.split(' '): + if cmd.find('/') > 0: + file_name = os.path.split(cmd)[1] + if len(file_name) == 0: + raise Exception('Failed to parse %s command component: %s' % (mksnapshot_name, cmd)) + if cmd.startswith('../../'): + file_path = os.path.realpath(os.path.join(build_dir, cmd)) + # Validate input file/path. + if not file_path.startswith(src_dir): + raise Exception('Invalid %s command input file: %s' % (mksnapshot_name, file_path)) + if not os.path.isfile(file_path): + raise Exception('Missing %s command input file: %s' % (mksnapshot_name, file_path)) + # Transfer input file. + copy_file(file_path, os.path.join(dst_dir, file_name), options.quiet) + cmd = file_name + parsed_cmd.append(cmd) + + # Write command-line arguments file. + write_file(os.path.join(dst_dir, 'mksnapshot_cmd.txt'), ' '.join(parsed_cmd)) + + # yapf: disable + files = [ + {'path': get_script_name('run_mksnapshot')}, + ] + # yapf: disable + + # Transfer other tools files. + copy_files_list(os.path.join(script_dir, 'distrib', 'tools'), output_dir, files) + + def normalize_headers(file, new_path=''): """ Normalize headers post-processing. Remove the path component from any project include directives. """ @@ -540,6 +665,12 @@ parser.add_option( dest='sandbox', default=False, help='include only the cef_sandbox static library (macOS and Windows only)') +parser.add_option( + '--tools', + action='store_true', + dest='tools', + default=False, + help='include only the tools') parser.add_option( '--ozone', action='store_true', @@ -597,10 +728,10 @@ if options.ozone and platform != 'linux': script_dir = os.path.dirname(__file__) # CEF root directory -cef_dir = os.path.abspath(os.path.join(script_dir, os.pardir)) +cef_dir = os.path.realpath(os.path.join(script_dir, os.pardir)) # src directory -src_dir = os.path.abspath(os.path.join(cef_dir, os.pardir)) +src_dir = os.path.realpath(os.path.join(cef_dir, os.pardir)) if not git.is_checkout(cef_dir): raise Exception('Not a valid checkout: %s' % (cef_dir)) @@ -665,6 +796,9 @@ elif options.client: elif options.sandbox: mode = 'sandbox' output_dir_name = output_dir_name + '_sandbox' +elif options.tools: + mode = 'tools' + output_dir_name = output_dir_name + '_tools' else: mode = 'standard' @@ -882,7 +1016,10 @@ if not options.nodocs: else: sys.stdout.write("ERROR: No docs generated.\n") -if platform == 'windows': +if mode == 'tools': + transfer_tools_files(script_dir, (build_dir_debug, build_dir_release), + output_dir) +elif platform == 'windows': libcef_dll = 'libcef.dll' libcef_dll_lib = '%s.lib' % libcef_dll libcef_dll_pdb = '%s.pdb' % libcef_dll