Extend the whole TensorFlow-Lite runtime infrastructure

- TFlite-runtime GPU delegate support
- TFlite-runtime external delegate support
- ARM Compute Library and ARMnn Delegate
- Whisper-TFlite updates.

buildroot-external/Config.in

@@ -1,7 +1,9 @@
 menu "Additional drivers, libraries and/or applications"
+	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/armnn/Config.in"
 	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/baresip/Config.in"
 	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/btspeaker/Config.in"
 	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/clog/Config.in"
+	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/compute-library/Config.in"
 	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/cpuinfo/Config.in"
 	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/deepfilternet-ladspa/Config.in"
 	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/docbook-xml/Config.in"
@@ -98,6 +100,7 @@ endmenu
 	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/lottie-qml/Config.in"
 	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/ncpamixer/Config.in"
 	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/neon2sse/Config.in"
+	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/opencl_headers/Config.in"
 	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/openfec/Config.in"
 	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/openfst/Config.in"
 	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/ovos-bus-server/Config.in"

buildroot-external/configs/rpi3_64-gui_defconfig

@@ -543,8 +543,10 @@ BR2_PACKAGE_HOST_PYTHON_SIX=y
 BR2_PACKAGE_HOST_PYTHON_XLRD=y
 BR2_PACKAGE_HOST_RUSTC=y
 BR2_PACKAGE_HOST_RUST=y
+BR2_PACKAGE_ARMNN=y
 BR2_PACKAGE_BARESIP=y
 BR2_PACKAGE_BTSPEAKER=y
+BR2_PACKAGE_COMPUTE_LIBRARY=y
 BR2_PACKAGE_DEEPFILTERNET_LADSPA=y
 BR2_PACKAGE_FANN=y
 BR2_PACKAGE_GROWDISK_SERVICE=y

buildroot-external/configs/rpi4_64-gui_defconfig

@@ -543,8 +543,10 @@ BR2_PACKAGE_HOST_PYTHON_SIX=y
 BR2_PACKAGE_HOST_PYTHON_XLRD=y
 BR2_PACKAGE_HOST_RUSTC=y
 BR2_PACKAGE_HOST_RUST=y
+BR2_PACKAGE_ARMNN=y
 BR2_PACKAGE_BARESIP=y
 BR2_PACKAGE_BTSPEAKER=y
+BR2_PACKAGE_COMPUTE_LIBRARY=y
 BR2_PACKAGE_DEEPFILTERNET_LADSPA=y
 BR2_PACKAGE_FANN=y
 BR2_PACKAGE_GROWDISK_SERVICE=y

buildroot-external/package/armnn/Config.in

@@ -0,0 +1,11 @@
+config BR2_PACKAGE_ARMNN
+	select BR2_PACKAGE_COMPUTE_LIBRARY
+	select BR2_PACKAGE_TENSORFLOW_LITE
+	bool "armnn"
+	help
+	  Arm NN is the most performant machine learning (ML) 
+	  inference engine for Android and Linux, accelerating 
+	  ML on Arm Cortex-A CPUs and Arm Mali GPUs. 
+
+	  https://github.com/ARM-software/armnn
+

buildroot-external/package/armnn/armnn.mk

@@ -0,0 +1,34 @@
+################################################################################
+#
+# armnn
+#
+################################################################################
+
+ARMNN_VERSION = v22.11.01
+ARMNN_SITE = $(call github,ARM-software,armnn,$(ARMNN_VERSION))
+ARMNN_LICENSE = MIT
+
+ARMNN_INSTALL_STAGING = YES
+ARMNN_DEPENDENCIES = host-pkgconf compute-library tensorflow-lite
+ARMNN_SUPPORTS_IN_SOURCE_BUILD = NO
+
+ARMNN_CONF_OPTS = \
+	-DCMAKE_C_FLAGS="$(TARGET_CFLAGS) -fPIC -Wno-error=missing-field-initializers" \
+	-DCMAKE_CXX_FLAGS="$(TARGET_CXXFLAGS) -fPIC -Wno-error=missing-field-initializers" \
+	-DCMAKE_SHARED_LINKER_FLAGS="$(TARGET_LDFLAGS) -fPIC -Wno-error=missing-field-initializers" \
+	-DCMAKE_POSITION_INDEPENDENT_CODE=ON \
+	-DBUILD_SHARED_LIBS=ON \
+	-DBUILD_TF_LITE_PARSER=1 \
+	-DBUILD_ARMNN_TFLITE_DELEGATE=ON \
+	-DARMCOMPUTENEON=ON \
+	-DARMCOMPUTECL=1 \
+	-DARMNNREF=1 \
+	-DARMCOMPUTE_ROOT="$(BUILD_DIR)/compute-library-$(COMPUTE_LIBRARY_VERSION)" \
+	-DARMCOMPUTE_BUILD_DIR="$(BUILD_DIR)/compute-library-$(COMPUTE_LIBRARY_VERSION)/buildroot-build" \
+	-DTENSORFLOW_ROOT="$(BUILD_DIR)/tensorflow-lite-$(TENSORFLOW_LITE_VERSION)" \
+	-DTF_LITE_SCHEMA_INCLUDE_PATH="$(BUILD_DIR)/tensorflow-lite-$(TENSORFLOW_LITE_VERSION)/tensorflow/lite/schema" \
+	-DTFLITE_LIB_ROOT="$(STAGING_DIR)/usr/lib" \
+	-DTfLite_Schema_INCLUDE_PATH="$(BUILD_DIR)/tensorflow-lite-$(TENSORFLOW_LITE_VERSION)/tensorflow/lite/schema" \
+	-DTfLite_LIB="$(STAGING_DIR)/usr/lib/libtensorflow-lite.so"
+
+$(eval $(cmake-package))

buildroot-external/package/compute-library/Config.in

@@ -0,0 +1,9 @@
+config BR2_PACKAGE_COMPUTE_LIBRARY
+	bool "compute-library"
+	help
+	  The Compute Library is a set of computer vision and 
+	  machine learning functions optimised for both Arm CPUs 
+	  and GPUs using SIMD technologies.
+
+	  https://github.com/ARM-software/ComputeLibrary
+

buildroot-external/package/compute-library/compute-library.mk

@@ -0,0 +1,65 @@
+################################################################################
+#
+# compute-library
+#
+################################################################################
+
+COMPUTE_LIBRARY_VERSION = v22.11
+COMPUTE_LIBRARY_SITE = $(call github,ARM-software,ComputeLibrary,$(COMPUTE_LIBRARY_VERSION))
+COMPUTE_LIBRARY_LICENSE = MIT
+COMPUTE_LIBRARY_LICENSE_FILES = LICENSE
+COMPUTE_LIBRARY_INSTALL_STAGING = YES
+
+COMPUTE_LIBRARY_DEPENDENCIES = \
+	host-scons \
+	host-pkgconf
+
+COMPUTE_LIBRARY_LDFLAGS = "$(TARGET_LDFLAGS)"
+COMPUTE_LIBRARY_CFLAGS = " $(TARGET_CFLAGS)"
+COMPUTE_LIBRARY_CXXFLAGS = "$(TARGET_CXXFLAGS)"
+
+COMPUTE_LIBRARY_SCONS_ENV = $(TARGET_CONFIGURE_OPTS)
+
+COMPUTE_LIBRARY_SCONS_OPTS = \
+	arch=arm64-v8a \
+	Werror=0 \
+	debug=0 \
+	asserts=0 \
+	neon=1 \
+        os=linux \
+	examples=0 \
+	build=cross_compile \
+	toolchain_prefix="" \
+	embed_kernels=1 \
+	extra_cxx_flags="-fPIC"
+
+COMPUTE_LIBRARY_SCONS_ENV += \
+	LDFLAGS=$(COMPUTE_LIBRARY_LDFLAGS) \
+	CFLAGS=$(COMPUTE_LIBRARY_CFLAGS) \
+	CCFLAGS=$(COMPUTE_LIBRARY_CFLAGS) \
+	CXXFLAGS=$(COMPUTE_LIBRARY_CXXFLAGS)
+
+define COMPUTE_LIBRARY_BUILD_CMDS
+	(cd $(@D); \
+	$(COMPUTE_LIBRARY_SCONS_ENV) \
+	$(SCONS) \
+	$(COMPUTE_LIBRARY_SCONS_OPTS))
+endef
+
+define COMPUTE_LIBRARY_INSTALL_STAGING_CMDS
+	(cd $(@D); \
+	$(COMPUTE_LIBRARY_SCONS_ENV) \
+	$(SCONS) \
+	$(COMPUTE_LIBRARY_SCONS_OPTS) \
+	install_dir="$(STAGING_DIR)/usr")
+endef
+
+define COMPUTE_LIBRARY_INSTALL_TARGET_CMDS
+	(cd $(@D); \
+	$(COMPUTE_LIBRARY_SCONS_ENV) \
+	$(SCONS) \
+	$(COMPUTE_LIBRARY_SCONS_OPTS) \
+	install_dir="$(TARGET_DIR)/usr")
+endef
+
+$(eval $(generic-package))

buildroot-external/package/opencl_headers/Config.in

@@ -0,0 +1,6 @@
+config BR2_PACKAGE_OPENCL_HEADERS
+	bool "opencl_headers"
+	help
+	  Header-only library for Khronos OpenCL-Headers.
+
+	  https://github.com/KhronosGroup/OpenCL-Headers

buildroot-external/package/opencl_headers/opencl_headers.hash

@@ -0,0 +1 @@
+sha256  ca8090359654e94f2c41e946b7e9d826253d795ae809ce7c83a7d3c859624693  opencl_headers-dcd5bede6859d26833cd85f0d6bbcee7382dc9b3.tar.gz

buildroot-external/package/opencl_headers/opencl_headers.mk

@@ -0,0 +1,23 @@
+################################################################################
+#
+# opencl_headers
+#
+################################################################################
+
+OPENCL_HEADERS_VERSION = dcd5bede6859d26833cd85f0d6bbcee7382dc9b3
+OPENCL_HEADERS_SITE = $(call github,KhronosGroup,OpenCL-Headers,$(OPENCL_HEADERS_VERSION))
+OPENCL_HEADERS_LICENSE = BSD-2-Clause
+OPENCL_HEADERS_LICENSE_FILES = LICENSE
+
+OPENCL_HEADERS_SUPPORTS_IN_SOURCE_BUILD = NO
+OPENCL_HEADERS_DEPENDENCIES = psimd
+
+OPENCL_HEADERS_CONF_OPTS = -DCMAKE_SYSTEM_NAME=Linux \
+		 -DCMAKE_SYSTEM_PROCESSOR=aarch64 \
+		 -DOPENCL_HEADERS_BUILD_TESTING=OFF \
+		 -DOPENCL_HEADERS_BUILD_CXX_TESTS=OFF
+
+OPENCL_HEADERS_INSTALL_STAGING = YES
+OPENCL_HEADERS_INSTALL_TARGET = NO
+
+$(eval $(cmake-package))

buildroot-external/package/tensorflow-lite/Config.in

@@ -16,6 +16,8 @@ config BR2_PACKAGE_TENSORFLOW_LITE
         select BR2_PACKAGE_NEON2SSE
         select BR2_PACKAGE_RUY
 	select BR2_PACKAGE_XNNPACK
+	select BR2_PACKAGE_OPENCL_HEADERS
+	select BR2_PACKAGE_VULKAN_HEADERS
         help
           Tensorflow Lite dynamic library and headers: Inference engine to run previously trained machine learning models.
 

buildroot-external/package/tensorflow-lite/tensorflow-lite.mk

@@ -24,7 +24,9 @@ TENSORFLOW_LITE_DEPENDENCIES += \
         libabseil-cpp \
         neon2sse \
         ruy \
-	xnnpack
+	xnnpack \
+	opencl_headers \
+	vulkan-headers
 
 TENSORFLOW_LITE_CONF_OPTS = \
 	-DCMAKE_C_FLAGS="$(TARGET_CFLAGS) -funsafe-math-optimizations \
@@ -50,17 +52,25 @@ TENSORFLOW_LITE_CONF_OPTS = \
         -DFlatBuffers_DIR=$(STAGING_DIR)/usr/lib/cmake/flatbuffers \
         -DNEON_2_SSE_DIR=$(STAGING_DIR)/usr/lib/cmake/NEON_2_SSE \
         -DTFLITE_ENABLE_EXTERNAL_DELEGATE=ON \
-        -DTFLITE_ENABLE_GPU=OFF \
+        -DTFLITE_ENABLE_GPU=ON \
         -DTFLITE_ENABLE_INSTALL=ON \
         -DTFLITE_ENABLE_MMAP=ON \
         -DTFLITE_ENABLE_NNAPI=ON \
         -DTFLITE_ENABLE_RUY=ON \
         -DTFLITE_ENABLE_XNNPACK=ON
 
-TENSORFLOW_LITE_MAKE_OPTS += _pywrap_tensorflow_interpreter_wrapper
+TENSORFLOW_LITE_MAKE_OPTS += _pywrap_tensorflow_interpreter_wrapper benchmark_model
+
+TENSORFLOW_LITE_POST_INSTALL_STAGING_HOOKS = TENSORFLOW_LITE_INSTALL_VERSION_HEADER
 
 TENSORFLOW_LITE_POST_INSTALL_TARGET_HOOKS = TENSORFLOW_LITE_INSTALL_TFLITE_RUNTIME
 
+define TENSORFLOW_LITE_INSTALL_VERSION_HEADER
+	mkdir -p  $(STAGING_DIR)/usr/include/tensorflow/core/public
+	$(INSTALL) -D -m 644  $(@D)/tensorflow/core/public/version.h \
+		$(STAGING_DIR)/usr/include/tensorflow/core/public/
+endef
+
 define TENSORFLOW_LITE_INSTALL_TFLITE_RUNTIME
 
 	mkdir -p $(TARGET_DIR)/usr/lib/python$(PYTHON3_VERSION_MAJOR)/site-packages/tflite_runtime

buildroot-external/package/whisper-tflite/0002-Add-multithread-support.patch

@@ -1,25 +0,0 @@
-From 4aa7797f8f043df84104b76ceca423832824ea13 Mon Sep 17 00:00:00 2001
-From: j1nx <p.steenbergen@j1nx.nl>
-Date: Fri, 9 Dec 2022 20:25:46 +0100
-Subject: [PATCH 1/1] Add multithread support
-
----
- tflite_minimal/minimal.cc | 2 ++
- 1 file changed, 2 insertions(+)
-
-diff --git a/tflite_minimal/minimal.cc b/tflite_minimal/minimal.cc
-index cd045e0..82e9b11 100644
---- a/tflite_minimal/minimal.cc
-+++ b/tflite_minimal/minimal.cc
-@@ -200,6 +200,8 @@ int main(int argc, char* argv[]) {
-   else if (argc == 3) {
-     memcpy(input, mel.data.data(), mel.n_mel*mel.n_len*sizeof(float));
-   }
-+  interpreter->SetNumThreads(2);
-+
-   // Fill input buffers
-   // TODO(user): Insert code to fill input tensors.
-   // Note: The buffer of the input tensor with index `i` of type T can
--- 
-2.34.1
-

buildroot-external/package/whisper-tflite/0002-Use-4-threads.patch

@@ -0,0 +1,24 @@
+From e2ebeaca6a16d981b0d130c06cfe38592623fbe3 Mon Sep 17 00:00:00 2001
+From: j1nx <p.steenbergen@j1nx.nl>
+Date: Fri, 20 Jan 2023 15:44:40 +0100
+Subject: [PATCH 1/1] Use 4 threads
+
+---
+ tflite_minimal/minimal.cc | 1 +
+ 1 file changed, 1 insertion(+)
+
+diff --git a/tflite_minimal/minimal.cc b/tflite_minimal/minimal.cc
+index cd045e0..88e21fe 100644
+--- a/tflite_minimal/minimal.cc
++++ b/tflite_minimal/minimal.cc
+@@ -186,6 +186,7 @@ int main(int argc, char* argv[]) {
+   tflite::InterpreterBuilder builder(*model, resolver);
+   std::unique_ptr<tflite::Interpreter> interpreter;
+   builder(&interpreter);
++  interpreter->SetNumThreads(4);
+   TFLITE_MINIMAL_CHECK(interpreter != nullptr);
+ 
+   // Allocate tensor buffers.
+-- 
+2.34.1
+

buildroot-external/package/whisper-tflite/0003-Multithread-support.patch

@@ -1,25 +0,0 @@
-From 624f7abc65cf1dee85d803c398b36a9b35106720 Mon Sep 17 00:00:00 2001
-From: j1nx <p.steenbergen@j1nx.nl>
-Date: Fri, 20 Jan 2023 14:22:23 +0100
-Subject: [PATCH 1/1] Multithread support
-
----
- tflite_minimal/minimal.cc | 2 ++
- 1 file changed, 2 insertions(+)
-
-diff --git a/tflite_minimal/minimal.cc b/tflite_minimal/minimal.cc
-index 82e9b11..e7d0187 100644
---- a/tflite_minimal/minimal.cc
-+++ b/tflite_minimal/minimal.cc
-@@ -186,6 +186,8 @@ int main(int argc, char* argv[]) {
-   tflite::InterpreterBuilder builder(*model, resolver);
-   std::unique_ptr<tflite::Interpreter> interpreter;
-   builder(&interpreter);
-+  const auto processor_count = std:: thread ::hardware_concurrency();
-+  interpreter->SetNumThreads(processor_count);
-   TFLITE_MINIMAL_CHECK(interpreter != nullptr);
- 
-   // Allocate tensor buffers.
--- 
-2.34.1
-

buildroot-patches/0023-Bump-libabseil-CXX-standard.patch

@@ -0,0 +1,31 @@
+From 6c4e6023d5b65447e23b737ebddff359668e64c4 Mon Sep 17 00:00:00 2001
+From: j1nx <p.steenbergen@j1nx.nl>
+Date: Wed, 25 Jan 2023 14:13:37 +0100
+Subject: [PATCH 1/1] Bump libabseil CXX standard
+
+---
+ package/libabseil-cpp/libabseil-cpp.mk | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/package/libabseil-cpp/libabseil-cpp.mk b/package/libabseil-cpp/libabseil-cpp.mk
+index 7c6bda7659..61ef26c11a 100644
+--- a/package/libabseil-cpp/libabseil-cpp.mk
++++ b/package/libabseil-cpp/libabseil-cpp.mk
+@@ -11,12 +11,12 @@ LIBABSEIL_CPP_LICENSE_FILES = LICENSE
+ LIBABSEIL_CPP_INSTALL_STAGING = YES
+ 
+ LIBABSEIL_CPP_CONF_OPTS = \
+-	-DCMAKE_CXX_STANDARD=11 \
++	-DCMAKE_CXX_STANDARD=17 \
+ 	-DABSL_ENABLE_INSTALL=ON \
+ 	-DABSL_USE_GOOGLETEST_HEAD=OFF
+ 
+ HOST_LIBABSEIL_CPP_CONF_OPTS = \
+-	-DCMAKE_CXX_STANDARD=11 \
++	-DCMAKE_CXX_STANDARD=17 \
+ 	-DABSL_ENABLE_INSTALL=ON \
+ 	-DABSL_USE_GOOGLETEST_HEAD=OFF
+ 
+-- 
+2.34.1
+