From d2059c5531a3b457a77e79731afa717347687457 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20=C4=8Cerm=C3=A1k?= Date: Wed, 20 Dec 2023 17:41:39 +0100 Subject: [PATCH] Add script for checking resulting kernel config (#3006) There is bunch of kernel config options that are not propagated correctly to the kernel configuration after fragments are merged and processed by Kconfig. Current Buildroot tools are not good at discovering these - while we cleaned up most inconsistencies by using linux-diff-config and output from the merge_config.sh script, there are still options that were removed or get a different value than intended because of dependencies, etc. This commit adds a Python script that is using Kconfiglib to parse current kernel's Kconfig files and the generated .config and compare the requested values from individual kernel config fragments. The script can be used manually by running `make linux-check-dotconfig` from the buildroot directory (with path to BR2_EXTERNAL directory set) and it's called also from the CI, where it generates Github Workflow warning annotations when some of the values are not present or when set incorrectly. The kconfiglib.py is checked-in to the repo as well, because the library is currently abandoned on PyPI and packaged version has a bug that causes errors parsing Kconfigs in newer Linux versions, fixed in outstanding pull request ulfalizer/Kconfiglib#119 - so version from this PR is used here. If pypi/support#2526 is ever resolved, we could remove it from our repo and use pip for installing the package as a requirement during build of the build container. --- buildroot-external/external.mk | 11 + buildroot-external/scripts/check-dotconfig.py | 128 + buildroot-external/scripts/kconfiglib.py | 7174 +++++++++++++++++ 3 files changed, 7313 insertions(+) create mode 100755 buildroot-external/scripts/check-dotconfig.py create mode 100644 buildroot-external/scripts/kconfiglib.py diff --git a/buildroot-external/external.mk b/buildroot-external/external.mk index abf0b74b..df052999 100644 --- a/buildroot-external/external.mk +++ b/buildroot-external/external.mk @@ -1 +1,12 @@ include $(sort $(wildcard $(BR2_EXTERNAL_OPENVOICEOS_PATH)/package/*/*.mk)) + +.PHONY: linux-check-dotconfig +linux-check-dotconfig: linux-check-configuration-done + CC=$(TARGET_CC) LD=$(TARGET_LD) srctree=$(LINUX_SRCDIR) \ + ARCH=$(if $(BR2_x86_64),x86,$(if $(BR2_arm)$(BR2_aarch64),arm,$(ARCH))) \ + SRCARCH=$(if $(BR2_x86_64),x86,$(if $(BR2_arm)$(BR2_aarch64),arm,$(ARCH))) \ + $(BR2_EXTERNAL_HASSOS_PATH)/scripts/check-dotconfig.py \ + $(BR2_CHECK_DOTCONFIG_OPTS) \ + --src-kconfig $(LINUX_SRCDIR)Kconfig \ + --actual-config $(LINUX_SRCDIR).config \ + $(shell echo $(BR2_LINUX_KERNEL_CONFIG_FRAGMENT_FILES)) diff --git a/buildroot-external/scripts/check-dotconfig.py b/buildroot-external/scripts/check-dotconfig.py new file mode 100755 index 00000000..6e5cc783 --- /dev/null +++ b/buildroot-external/scripts/check-dotconfig.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python + +import argparse +from collections import namedtuple +import re + +from kconfiglib import Kconfig + + +# Can be either "CONFIG_OPTION=(y|m|n)" or "# CONFIG_OPTION is not set" +regex = re.compile( + r"^(CONFIG_(?P[A-Z0-9_]+)=(?P[mny])" + r"|# CONFIG_(?P[A-Z0-9_]+) is not set)$" +) + +# use namedtuple as a lightweight representation of fragment-defined options +OptionValue = namedtuple("OptionValue", ["option", "value", "file", "line"]) + + +def parse_fragment( + filename: str, strip_path_prefix: str = None +) -> dict[str, OptionValue]: + """ + Parse Buildroot Kconfig fragment and return dict of OptionValue objects. + """ + options: dict[str, OptionValue] = {} + + with open(filename) as f: + if strip_path_prefix and filename.startswith(strip_path_prefix): + filename = filename[len(strip_path_prefix) :] + + for line_number, line in enumerate(f, 1): + if matches := re.match(regex, line): + if matches["option_unset"]: + value = OptionValue( + matches["option_unset"], None, filename, line_number + ) + options.update({matches.group("option_unset"): value}) + else: + value = OptionValue( + matches["option_set"], matches["value"], filename, line_number + ) + options.update({matches.group("option_set"): value}) + + return options + + +def _format_message( + message: str, file: str, line: int, github_format: bool = False +) -> str: + """ + Format message with source file and line number. + """ + if github_format: + return f"::warning file={file},line={line}::{message}" + return f"{message} (defined in {file}:{line})" + + +def compare_configs( + expected_options: dict[str, OptionValue], + kconfig: Kconfig, + github_format: bool = False, +) -> None: + """ + Compare dictionary of expected options with actual Kconfig representation. + """ + for option, spec in expected_options.items(): + if option not in kconfig.syms: + print( + _format_message( + f"{option}={spec.value} not found", + file=spec.file, + line=spec.line, + github_format=github_format, + ) + ) + elif (val := kconfig.syms[option].str_value) != spec.value: + if spec.value is None and val == "n": + continue + print( + _format_message( + f"{option}={spec.value} requested, actual = {val}", + file=spec.file, + line=spec.line, + github_format=github_format, + ) + ) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "--src-kconfig", help="Path to top-level Kconfig file", required=True + ) + parser.add_argument( + "--actual-config", + help="Path to config with actual config values (.config)", + required=True, + ) + parser.add_argument( + "--github-format", + action="store_true", + help="Use Github Workflow commands output format", + ) + parser.add_argument( + "-s", + "--strip-path-prefix", + help="Path prefix to strip in the output from config fragment paths", + ) + parser.add_argument("fragments", nargs="+", help="Paths to source config fragments") + + args = parser.parse_args() + + expected_options: dict[str, OptionValue] = {} + + for f in args.fragments: + expected_options.update( + parse_fragment(f, strip_path_prefix=args.strip_path_prefix) + ) + + kconfig = Kconfig(args.src_kconfig, warn_to_stderr=False) + kconfig.load_config(args.actual_config) + + compare_configs(expected_options, kconfig, github_format=args.github_format) + + +if __name__ == "__main__": + main() diff --git a/buildroot-external/scripts/kconfiglib.py b/buildroot-external/scripts/kconfiglib.py new file mode 100644 index 00000000..ccef1239 --- /dev/null +++ b/buildroot-external/scripts/kconfiglib.py @@ -0,0 +1,7174 @@ +# Copyright (c) 2011-2019, Ulf Magnusson +# SPDX-License-Identifier: ISC + +""" +Overview +======== + +Kconfiglib is a Python 2/3 library for scripting and extracting information +from Kconfig (https://www.kernel.org/doc/Documentation/kbuild/kconfig-language.txt) +configuration systems. + +See the homepage at https://github.com/ulfalizer/Kconfiglib for a longer +overview. + +Since Kconfiglib 12.0.0, the library version is available in +kconfiglib.VERSION, which is a (, , ) tuple, e.g. +(12, 0, 0). + + +Using Kconfiglib on the Linux kernel with the Makefile targets +============================================================== + +For the Linux kernel, a handy interface is provided by the +scripts/kconfig/Makefile patch, which can be applied with either 'git am' or +the 'patch' utility: + + $ wget -qO- https://raw.githubusercontent.com/ulfalizer/Kconfiglib/master/makefile.patch | git am + $ wget -qO- https://raw.githubusercontent.com/ulfalizer/Kconfiglib/master/makefile.patch | patch -p1 + +Warning: Not passing -p1 to patch will cause the wrong file to be patched. + +Please tell me if the patch does not apply. It should be trivial to apply +manually, as it's just a block of text that needs to be inserted near the other +*conf: targets in scripts/kconfig/Makefile. + +Look further down for a motivation for the Makefile patch and for instructions +on how you can use Kconfiglib without it. + +If you do not wish to install Kconfiglib via pip, the Makefile patch is set up +so that you can also just clone Kconfiglib into the kernel root: + + $ git clone git://github.com/ulfalizer/Kconfiglib.git + $ git am Kconfiglib/makefile.patch (or 'patch -p1 < Kconfiglib/makefile.patch') + +Warning: The directory name Kconfiglib/ is significant in this case, because +it's added to PYTHONPATH by the new targets in makefile.patch. + +The targets added by the Makefile patch are described in the following +sections. + + +make kmenuconfig +---------------- + +This target runs the curses menuconfig interface with Python 3. As of +Kconfiglib 12.2.0, both Python 2 and Python 3 are supported (previously, only +Python 3 was supported, so this was a backport). + + +make guiconfig +-------------- + +This target runs the Tkinter menuconfig interface. Both Python 2 and Python 3 +are supported. To change the Python interpreter used, pass +PYTHONCMD= to 'make'. The default is 'python'. + + +make [ARCH=] iscriptconfig +-------------------------------- + +This target gives an interactive Python prompt where a Kconfig instance has +been preloaded and is available in 'kconf'. To change the Python interpreter +used, pass PYTHONCMD= to 'make'. The default is 'python'. + +To get a feel for the API, try evaluating and printing the symbols in +kconf.defined_syms, and explore the MenuNode menu tree starting at +kconf.top_node by following 'next' and 'list' pointers. + +The item contained in a menu node is found in MenuNode.item (note that this can +be one of the constants kconfiglib.MENU and kconfiglib.COMMENT), and all +symbols and choices have a 'nodes' attribute containing their menu nodes +(usually only one). Printing a menu node will print its item, in Kconfig +format. + +If you want to look up a symbol by name, use the kconf.syms dictionary. + + +make scriptconfig SCRIPT=