distrib: Add new tools distribution for mksnapshot (see #3734)

tools/automate/automate-git.py

@@ -461,6 +461,7 @@ def check_pattern_matches(output_file=None):
       # Don't continue when we know the build will be wrong.
       sys.exit(1)
 
+
 def invalid_options_combination(a, b):
   print("Invalid combination of options: '%s' and '%s'" % (a, b))
   parser.print_help(sys.stderr)
@@ -767,6 +768,18 @@ parser.add_option(
     dest='sandboxdistribonly',
     default=False,
     help='Create a cef_sandbox static library distribution only.')
+parser.add_option(
+    '--tools-distrib',
+    action='store_true',
+    dest='toolsdistrib',
+    default=False,
+    help='Create a tools distribution.')
+parser.add_option(
+    '--tools-distrib-only',
+    action='store_true',
+    dest='toolsdistribonly',
+    default=False,
+    help='Create a tools distribution only.')
 parser.add_option(
     '--no-distrib-docs',
     action='store_true',
@@ -829,7 +842,7 @@ if (options.forcecleandeps and options.fastupdate):
 if (options.noreleasebuild and \
      (options.minimaldistrib or options.minimaldistribonly or \
       options.clientdistrib or options.clientdistribonly)) or \
-   (options.minimaldistribonly + options.clientdistribonly + options.sandboxdistribonly > 1):
+   (options.minimaldistribonly + options.clientdistribonly + options.sandboxdistribonly + options.toolsdistribonly > 1):
   print('Invalid combination of options.')
   parser.print_help(sys.stderr)
   sys.exit(1)
@@ -1450,6 +1463,8 @@ if not options.nodistrib and (chromium_checkout_changed or \
     distrib_types.append('client')
   elif options.sandboxdistribonly:
     distrib_types.append('sandbox')
+  elif options.toolsdistribonly:
+    distrib_types.append('tools')
   else:
     distrib_types.append('standard')
     if options.minimaldistrib:
@@ -1458,6 +1473,8 @@ if not options.nodistrib and (chromium_checkout_changed or \
       distrib_types.append('client')
     if options.sandboxdistrib:
       distrib_types.append('sandbox')
+    if options.toolsdistrib:
+      distrib_types.append('tools')
 
   cef_tools_dir = os.path.join(cef_src_dir, 'tools')
 
@@ -1482,6 +1499,8 @@ if not options.nodistrib and (chromium_checkout_changed or \
       path += ' --client'
     elif type == 'sandbox':
       path += ' --sandbox'
+    elif type == 'tools':
+      path += ' --tools'
 
     if first_type:
       if options.nodistribdocs:

tools/distrib/README.tools.txt

@@ -0,0 +1,58 @@
+CONTENTS
+--------
+
+Debug       Contains the Debug build of tools.
+
+Release     Contains the Release build of tools.
+
+
+IMPORTANT NOTE
+--------------
+
+CEF/Chromium builds are created using the following host architectures:
+
+- Linux:   x86-64 (Intel/AMD)
+- Windows: x86-64 (Intel/AMD)
+- MacOS:   ARM64 (Apple Silicon)
+
+Binaries in this tools package must be run on the supported host OS/architecture
+even in cases where the output targets a different architecture.
+
+For example, files targeting a MacOS 64-bit (Intel) application must be created
+on a MacOS ARM64 (Apple Silicon) host system using the MacOS 64-bit (Intel)
+tools distribution.
+
+
+USAGE
+-----
+
+Start with the required host system and the tools distribution that matches your
+application's target OS/architecture and CEF version.
+
+Custom V8 Snapshots
+
+Custom startup snapshots [https://v8.dev/blog/custom-startup-snapshots] can be
+used to speed up V8/JavaScript load time in the renderer process. With CEF this
+works as follows:
+
+1. Generate a single JavaScript file that contains custom startup data. For
+   example, using https://github.com/atom/electron-link.
+
+2. Execute the `run_mksnapshot` script to create a `v8_context_snapshot.bin`
+   file containing the custom data in addition to the default V8 data.
+
+   Example:
+   % run_mksnapshot Release /path/to/snapshot.js
+
+   Note that bin file names include an architecture component on MacOS
+   (e.g. `v8_context_snapshot.[arm64|x86_64].bin`)
+
+3. Replace the existing `v8_context_snapshot.bin` file in the installation
+   folder or app bundle.
+
+4. Run the application and verify in DevTools that the custom startup data
+   exists. For example, electron-link adds a global `snapshotResult` object.
+
+Please visit the CEF Website for additional usage information.
+
+https://bitbucket.org/chromiumembedded/cef/

tools/distrib/tools/run_mksnapshot.bat

@@ -0,0 +1,66 @@
+@echo off
+:: Copyright (c) 2024 The Chromium Embedded Framework Authors. All rights
+:: reserved. Use of this source code is governed by a BSD-style license
+:: that can be found in the LICENSE file.
+
+set RC=
+
+setlocal
+
+if not "%1" == "Debug" (
+  if not "%1" == "Release" (
+    echo Usage: run_mksnapshot.bat [Debug^|Release] path\to\snapshot.js
+    set ERRORLEVEL=1
+    goto end
+  )
+)
+
+set SCRIPT_DIR=%~dp0
+set BIN_DIR=%SCRIPT_DIR%%~1
+
+if not exist "%BIN_DIR%" (
+  echo %BIN_DIR% directory not found
+  set ERRORLEVEL=1
+  goto end
+)
+
+set CMD_FILE=mksnapshot_cmd.txt
+
+if not exist "%BIN_DIR%\%CMD_FILE%" (
+  echo %BIN_DIR%\%CMD_FILE% file not found
+  set ERRORLEVEL=1
+  goto end
+)
+
+cd "%BIN_DIR%"
+
+:: Read %CMD_FILE% into a local variable.
+set /p CMDLINE=<%CMD_FILE%
+
+:: Generate snapshot_blob.bin.
+echo Running mksnapshot...
+call mksnapshot %CMDLINE% %2
+
+set OUT_FILE=v8_context_snapshot.bin
+
+:: Generate v8_context_snapshot.bin.
+echo Running v8_context_snapshot_generator...
+call v8_context_snapshot_generator --output_file=%OUT_FILE%
+
+if not exist "%BIN_DIR%\%OUT_FILE%" (
+  echo Failed
+  set ERRORLEVEL=1
+  goto end
+)
+
+echo Success! Created %BIN_DIR%\%OUT_FILE%
+
+:end
+endlocal & set RC=%ERRORLEVEL%
+goto omega
+
+:returncode
+exit /B %RC%
+
+:omega
+call :returncode %RC%

tools/distrib/tools/run_mksnapshot.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+# Copyright (c) 2024 The Chromium Embedded Framework Authors. All rights
+# reserved. Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file.
+
+if [ "$1" != "Debug" ] && [ "$1" != "Release" ]; then
+  echo 'Usage: run_mksnapshot.sh [Debug|Release] /path/to/snapshot.js'
+  exit 1
+fi
+
+SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
+BIN_DIR=$SCRIPT_DIR/$1
+
+if [ ! -d "$BIN_DIR" ]; then
+  echo "$BIN_DIR directory not found."
+  exit 1
+fi
+
+CMD_FILE=mksnapshot_cmd.txt
+
+if [ ! -f "$BIN_DIR/$CMD_FILE" ]; then
+  echo "$BIN_DIR/$CMD_FILE file not found."
+  exit 1
+fi
+
+pushd "$BIN_DIR" > /dev/null
+
+# Read $CMD_FILE into an argument array.
+IFS=' ' read -r -a args < $CMD_FILE
+
+if [ "$(uname)" == "Darwin" ]; then
+  # Execution of tools binaries downloaded on MacOS may be blocked
+  # by Apple Security settings with a message like "<binary> can't
+  # be opened because Apple cannot check it for malicious software."
+  # Remove that block here.
+  xattr -c ./mksnapshot
+  xattr -c ./v8_context_snapshot_generator
+fi
+
+# Generate snapshot_blob.bin.
+echo 'Running mksnapshot...'
+./mksnapshot "${args[@]}" "${@:2}"
+
+# Determine the architecture suffix, if any.
+suffix=''
+if [ "$(uname)" == "Darwin" ]; then
+  value='--target_arch=arm64'
+  if [[ " ${args[*]} " =~ [[:space:]]${value}[[:space:]] ]]; then
+    suffix='.arm64'
+  else
+    suffix='.x86_64'
+  fi
+fi
+
+OUT_FILE=v8_context_snapshot${suffix}.bin
+
+# Generate v8_context_snapshot.bin.
+echo 'Running v8_context_snapshot_generator...'
+./v8_context_snapshot_generator --output_file=$OUT_FILE
+
+popd > /dev/null
+
+if [ -f "$BIN_DIR/$OUT_FILE" ]; then
+  echo "Success! Created $BIN_DIR/$OUT_FILE"
+else
+  echo "Failed"
+  exit 1
+fi

tools/make_distrib.py

@@ -122,7 +122,7 @@ def create_readme():
 
   # format the file
   data = header_data + '\n\n' + mode_data
-  if mode != 'sandbox':
+  if mode != 'sandbox' and mode != 'tools':
     data += '\n\n' + redistrib_data
   data += '\n\n' + footer_data
   data = data.replace('$CEF_URL$', cef_url)
@@ -165,6 +165,9 @@ def create_readme():
     distrib_type = 'Sandbox'
     distrib_desc = 'This distribution contains only the cef_sandbox static library. Please see\n' \
                    'the LICENSING section of this document for licensing terms and conditions.'
+  elif mode == 'tools':
+    distrib_type = 'Tools'
+    distrib_desc = 'This distribution contains additional tools for building CEF-based applications.'
 
   data = data.replace('$DISTRIB_TYPE$', distrib_type)
   data = data.replace('$DISTRIB_DESC$', distrib_desc)
@@ -232,6 +235,128 @@ def transfer_gypi_files(src_dir, gypi_paths, gypi_path_prefix, dst_dir, quiet):
     copy_file(src, dst, quiet)
 
 
+def extract_toolchain_cmd(build_dir,
+                          exe_name,
+                          require_toolchain,
+                          require_cmd=True):
+  """ Extract a toolchain command from the ninja configuration file. """
+  toolchain_ninja = os.path.join(build_dir, 'toolchain.ninja')
+  if not os.path.isfile(toolchain_ninja):
+    if not require_toolchain:
+      return None, None
+    raise Exception('Missing file: %s' % toolchain_ninja)
+
+  data = read_file(toolchain_ninja)
+
+  cmd = None
+  path = None
+
+  # Looking for a value like:
+  #   command = python3 ../../v8/tools/run.py ./exe_name --arg1 --arg2
+  # OR (for cross-compile):
+  #   command = python3 ../../v8/tools/run.py ./clang_arch1_arch2/exe_name --arg1 --arg2
+  findstr = '/%s ' % exe_name
+  start = data.find(findstr)
+  if start >= 0:
+    # Extract the command-line arguments.
+    after_start = start + len(findstr)
+    end = data.find('\n', after_start)
+    if end >= after_start:
+      cmd = data[after_start:end].strip()
+      print('%s command:' % exe_name, cmd)
+      if cmd != '' and not re.match(r"^[0-9a-zA-Z\_\- ./=]{1,}$", cmd):
+        cmd = None
+
+    # Extract the relative file path.
+    dot = start - 1
+    while data[dot].isalnum() or data[dot] == '_':
+      dot -= 1
+    path = data[dot + 1:start]
+    print('%s path:' % exe_name, path)
+    if path != '' and not re.match(r"^(win_)?clang_[0-9a-z_]{1,}$", path):
+      path = None
+
+  if require_cmd and (cmd is None or path is None):
+    raise Exception('Failed to extract %s command from %s' % (exe_name,
+                                                              toolchain_ninja))
+
+  return cmd, path
+
+
+def get_exe_name(exe_name):
+  return exe_name + ('.exe' if platform == 'windows' else '')
+
+
+def get_script_name(script_name):
+  return script_name + ('.bat' if platform == 'windows' else '.sh')
+
+
+def transfer_tools_files(script_dir, build_dirs, output_dir):
+  for build_dir in build_dirs:
+    is_debug = build_dir.find('Debug') >= 0
+    dst_dir_name = 'Debug' if is_debug else 'Release'
+    dst_dir = os.path.join(output_dir, dst_dir_name)
+
+    # Retrieve the binary path and command-line arguments.
+    # See issue #3734 for the expected format.
+    mksnapshot_name = 'mksnapshot'
+    tool_cmd, tool_dir = extract_toolchain_cmd(
+        build_dir, mksnapshot_name, require_toolchain=not options.allowpartial)
+    if tool_cmd is None:
+      sys.stdout.write("No %s build toolchain for %s.\n" % (dst_dir_name,
+                                                            mksnapshot_name))
+      continue
+
+    if options.allowpartial and not path_exists(
+        os.path.join(build_dir, tool_dir, get_exe_name(mksnapshot_name))):
+      sys.stdout.write("No %s build of %s.\n" % (dst_dir_name, mksnapshot_name))
+      continue
+
+    # yapf: disable
+    binaries = [
+        {'path': get_exe_name(mksnapshot_name)},
+        {'path': get_exe_name('v8_context_snapshot_generator')},
+    ]
+    # yapf: disable
+
+    # Transfer binaries.
+    copy_files_list(os.path.join(build_dir, tool_dir), dst_dir, binaries)
+
+    # Evaluate command-line arguments and remove relative paths. Copy any input files
+    # into the distribution.
+    # - Example input path : ../../v8/tools/builtins-pgo/profiles/x64-rl.profile
+    # - Example output path: gen/v8/embedded.S
+    parsed_cmd = []
+    for cmd in tool_cmd.split(' '):
+      if cmd.find('/') > 0:
+        file_name = os.path.split(cmd)[1]
+        if len(file_name) == 0:
+          raise Exception('Failed to parse %s command component: %s' % (mksnapshot_name, cmd))
+        if cmd.startswith('../../'):
+          file_path = os.path.realpath(os.path.join(build_dir, cmd))
+          # Validate input file/path.
+          if not file_path.startswith(src_dir):
+            raise Exception('Invalid %s command input file: %s' % (mksnapshot_name, file_path))
+          if not os.path.isfile(file_path):
+            raise Exception('Missing %s command input file: %s' % (mksnapshot_name, file_path))
+          # Transfer input file.
+          copy_file(file_path, os.path.join(dst_dir, file_name), options.quiet)
+        cmd = file_name
+      parsed_cmd.append(cmd)
+
+    # Write command-line arguments file.
+    write_file(os.path.join(dst_dir, 'mksnapshot_cmd.txt'), ' '.join(parsed_cmd))
+
+  # yapf: disable
+  files = [
+      {'path': get_script_name('run_mksnapshot')},
+  ]
+  # yapf: disable
+
+  # Transfer other tools files.
+  copy_files_list(os.path.join(script_dir, 'distrib', 'tools'), output_dir, files)
+
+
 def normalize_headers(file, new_path=''):
   """ Normalize headers post-processing. Remove the path component from any
       project include directives. """
@@ -540,6 +665,12 @@ parser.add_option(
     dest='sandbox',
     default=False,
     help='include only the cef_sandbox static library (macOS and Windows only)')
+parser.add_option(
+    '--tools',
+    action='store_true',
+    dest='tools',
+    default=False,
+    help='include only the tools')
 parser.add_option(
     '--ozone',
     action='store_true',
@@ -597,10 +728,10 @@ if options.ozone and platform != 'linux':
 script_dir = os.path.dirname(__file__)
 
 # CEF root directory
-cef_dir = os.path.abspath(os.path.join(script_dir, os.pardir))
+cef_dir = os.path.realpath(os.path.join(script_dir, os.pardir))
 
 # src directory
-src_dir = os.path.abspath(os.path.join(cef_dir, os.pardir))
+src_dir = os.path.realpath(os.path.join(cef_dir, os.pardir))
 
 if not git.is_checkout(cef_dir):
   raise Exception('Not a valid checkout: %s' % (cef_dir))
@@ -665,6 +796,9 @@ elif options.client:
 elif options.sandbox:
   mode = 'sandbox'
   output_dir_name = output_dir_name + '_sandbox'
+elif options.tools:
+  mode = 'tools'
+  output_dir_name = output_dir_name + '_tools'
 else:
   mode = 'standard'
 
@@ -882,7 +1016,10 @@ if not options.nodocs:
   else:
     sys.stdout.write("ERROR: No docs generated.\n")
 
-if platform == 'windows':
+if mode == 'tools':
+  transfer_tools_files(script_dir, (build_dir_debug, build_dir_release),
+                       output_dir)
+elif platform == 'windows':
   libcef_dll = 'libcef.dll'
   # yapf: disable
   binaries = [