[WIP] Build vosk-api andits dependencies from source

This so we can use the same package for any architecture in the future.
2022-12-20 15:58:35 +01:00 · 2022-12-20 15:58:35 +01:00 · 08e08a0c5b
parent 8a75b8e37a
commit 08e08a0c5b
22 changed files with 325 additions and 0 deletions
--- a/buildroot-external/Config.in
+++ b/buildroot-external/Config.in
@ -13,6 +13,7 @@ menu "Additional drivers, libraries and/or applications"
 	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/gemmlowp/Config.in"
 	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/growdisk-service/Config.in"
 	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/hostname-service/Config.in"
+	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/kaldi/Config.in"
 menu "KDE Framework and Plasma"
 	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/kf5-attica/Config.in"
 	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/kf5-breeze-icons/Config.in"
@ -94,6 +95,7 @@ endmenu
 	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/lottie-qml/Config.in"
 	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/ncpamixer/Config.in"
 	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/neon2sse/Config.in"
+	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/openfst/Config.in"
 	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/ovos-dashboard/Config.in"
 	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/ovos-shell/Config.in"
 	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/ovos-splash/Config.in"
@ -110,6 +112,7 @@ endmenu
 	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/userland-tools/Config.in"
 	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/virtual-touch/Config.in"
 	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/vocalfusion/Config.in"
+	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/vosk-api/Config.in"
 	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/whisper-tflite/Config.in"
 	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/whispercpp/Config.in"
 	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/wifi-connect/Config.in"
@ -238,6 +241,7 @@ menu "Additional external python modules"
 	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/python-source/Config.in"
 	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/python-speech2text/Config.in"
 	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/python-speechrecognition/Config.in"
+	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/python-srt/Config.in"
 	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/python-stopwordsiso/Config.in"
 	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/python-sysv-ipc/Config.in"
 	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/python-tailhead/Config.in"
@ -252,6 +256,7 @@ menu "Additional external python modules"
 	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/python-url-normalize/Config.in"
 	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/python-user-agents/Config.in"
 	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/python-vlc/Config.in"
+	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/python-vosk-api/Config.in"
 	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/python-webrtcvad/Config.in"
 	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/python-wheel/Config.in"
 	source "$BR2_EXTERNAL_OPENVOICEOS_PATH/package/python-wikipedia-api/Config.in"
--- a/buildroot-external/configs/rpi3_64-gui_defconfig
+++ b/buildroot-external/configs/rpi3_64-gui_defconfig
@ -683,6 +683,7 @@ BR2_PACKAGE_PYTHON_TUTUBO=y
 BR2_PACKAGE_PYTHON_TZLOCAL=y
 BR2_PACKAGE_PYTHON_URL_NORMALIZE=y
 BR2_PACKAGE_PYTHON_VLC=y
+BR2_PACKAGE_PYTHON_VOSK_API=y
 BR2_PACKAGE_PYTHON_WEBRTCVAD=y
 BR2_PACKAGE_PYTHON_WHEEL=y
 BR2_PACKAGE_PYTHON_WIKIPEDIA_FOR_HUMANS=y
--- a/buildroot-external/configs/rpi4_64-gui_defconfig
+++ b/buildroot-external/configs/rpi4_64-gui_defconfig
@ -683,6 +683,7 @@ BR2_PACKAGE_PYTHON_TUTUBO=y
 BR2_PACKAGE_PYTHON_TZLOCAL=y
 BR2_PACKAGE_PYTHON_URL_NORMALIZE=y
 BR2_PACKAGE_PYTHON_VLC=y
+BR2_PACKAGE_PYTHON_VOSK_API=y
 BR2_PACKAGE_PYTHON_WEBRTCVAD=y
 BR2_PACKAGE_PYTHON_WHEEL=y
 BR2_PACKAGE_PYTHON_WIKIPEDIA_FOR_HUMANS=y
--- a/buildroot-external/package/kaldi/0001-Fix-cmake-install-rpath.patch
+++ b/buildroot-external/package/kaldi/0001-Fix-cmake-install-rpath.patch
@ -0,0 +1,25 @@
+From 8651f18563ae11be6f26edc35afce32e884b2ed5 Mon Sep 17 00:00:00 2001
+From: j1nx <p.steenbergen@j1nx.nl>
+Date: Mon, 19 Dec 2022 10:20:48 +0100
+Subject: [PATCH 1/1] Fix cmake install rpath
+
+---
+ CMakeLists.txt | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index e0ca3ea28..b7aa22fa9 100644
+--- a/CMakeLists.txt
+++ b/CMakeLists.txt
+@@ -52,7 +52,7 @@ if(BUILD_SHARED_LIBS)
+     elseif(APPLE)
+         set(CMAKE_INSTALL_RPATH "@loader_path")
+     else()
+-        set(CMAKE_INSTALL_RPATH "$ORIGIN;$ORIGIN/../lib;$ORIGIN/../../tools/openfst/lib")
+        set(CMAKE_INSTALL_RPATH ${CMAKE_INSTALL_FULL_LIBDIR})
+     endif()
+ endif()
+ 
+-- 
+2.34.1
+
--- a/buildroot-external/package/kaldi/0002-Fix-cblas-linking-issue-with-cmake.patch
+++ b/buildroot-external/package/kaldi/0002-Fix-cblas-linking-issue-with-cmake.patch
@ -0,0 +1,25 @@
+From a35d6219f9dba75a99dd90f3c02666a6c017c8fc Mon Sep 17 00:00:00 2001
+From: j1nx <p.steenbergen@j1nx.nl>
+Date: Mon, 19 Dec 2022 10:05:44 +0100
+Subject: [PATCH 1/1] Fix cblas linking issue with cmake
+
+---
+ cmake/gen_cmake_skeleton.py | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/cmake/gen_cmake_skeleton.py b/cmake/gen_cmake_skeleton.py
+index 5925c6369..88df4a0b6 100644
+--- a/cmake/gen_cmake_skeleton.py
+++ b/cmake/gen_cmake_skeleton.py
+@@ -269,7 +269,7 @@ class CMakeListsLibrary(object):
+ 
+         if len(self.depends) > 0:
+             ret.append("target_link_libraries(" + self.target_name + " PUBLIC")
+-            for d in self.depends:
+            for d in self.depends + ['-lcblas', '-llapack'] :
+                 ret.append("    " + d)
+             ret.append(")\n")
+ 
+-- 
+2.34.1
+
--- a/buildroot-external/package/kaldi/Config.in
+++ b/buildroot-external/package/kaldi/Config.in
@ -0,0 +1,9 @@
+config BR2_PACKAGE_KALDI
+	bool "kaldi"
+	select BR2_PACKAGE_LAPACK
+	select BR2_PACKAGE_OPENFST
+	help
+	  Kaldi Speech Recognition Toolkit
+
+	  https://github.com/kaldi-asr/kaldi
+
--- a/buildroot-external/package/kaldi/kaldi.hash
+++ b/buildroot-external/package/kaldi/kaldi.hash
@ -0,0 +1,2 @@
+# Locally computed
+sha256  f47a996af546884a8e1cb961323b796c5c238c1bbeabaf657bb300b760e431b2  kaldi-93ef0019b847272a239fbb485ef97f29feb1d587.tar.gz
--- a/buildroot-external/package/kaldi/kaldi.mk
+++ b/buildroot-external/package/kaldi/kaldi.mk
@ -0,0 +1,20 @@
+################################################################################
+#
+# kaldi
+#
+################################################################################
+
+KALDI_VERSION = 93ef0019b847272a239fbb485ef97f29feb1d587
+KALDI_SITE = $(call github,alphacep,kaldi,$(KALDI_VERSION))
+KALDI_LICENSE = Apache License 2.0
+
+KALDI_INSTALL_STAGING = YES
+KALDI_DEPENDENCIES = host-pkgconf openfst lapack
+KALDI_SUPPORTS_IN_SOURCE_BUILD = NO
+
+KALDI_CONF_OPTS = \
+	-DMATHLIB=OpenBLAS \
+	-DKALDI_BUILD_TEST=OFF \
+	-DBUILD_SHARED_LIBS=ON
+
+$(eval $(cmake-package))
--- a/buildroot-external/package/openfst/Config.in
+++ b/buildroot-external/package/openfst/Config.in
@ -0,0 +1,7 @@
+config BR2_PACKAGE_OPENFST
+	bool "openfst"
+	help
+	  Port of the OpenFST library to Windows
+
+	  https://github.com/kkm000/openfst
+
--- a/buildroot-external/package/openfst/openfst.hash
+++ b/buildroot-external/package/openfst/openfst.hash
@ -0,0 +1,2 @@
+# Locally computed
+sha256  a7bac90acb9a12fdfe50c240c9bb730a7bcb5c3b3ac32017c4d27177b059680a  openfst-7dfd808194105162f20084bb4d8e4ee4b65266d5.tar.gz
--- a/buildroot-external/package/openfst/openfst.mk
+++ b/buildroot-external/package/openfst/openfst.mk
@ -0,0 +1,23 @@
+################################################################################
+#
+# openfst
+#
+################################################################################
+
+OPENFST_VERSION = 7dfd808194105162f20084bb4d8e4ee4b65266d5
+OPENFST_SITE = $(call github,alphacep,openfst,$(OPENFST_VERSION))
+OPENFST_LICENSE = Apache License 2.0
+
+OPENFST_INSTALL_STAGING = YES
+
+OPENFST_AUTORECONF = YES
+OPENFST_CONF_OPTS = \
+	--enable-static \
+	--enable-shared \
+	--enable-far \
+	--enable-ngram-fsts \
+	--enable-lookahead-fsts \
+	--with-pic \
+	--disable-bin
+
+$(eval $(autotools-package))
--- a/buildroot-external/package/python-srt/Config.in
+++ b/buildroot-external/package/python-srt/Config.in
@ -0,0 +1,7 @@
+config BR2_PACKAGE_PYTHON_SRT
+	bool "python-srt"
+	help
+	  A tiny library for parsing, modifying, and composing SRT
+	  files.
+
+	  https://github.com/cdown/srt
--- a/buildroot-external/package/python-srt/python-srt.hash
+++ b/buildroot-external/package/python-srt/python-srt.hash
@ -0,0 +1,5 @@
+# md5, sha256 from https://pypi.org/pypi/srt/json
+md5  3b68be7c46ec6152123fd801f519a63d  srt-3.5.2.tar.gz
+sha256  7aa4ad5ce4126d3f53b3e7bc4edaa86653d0378bf1c0b1ab8c59f5ab41384450  srt-3.5.2.tar.gz
+# Locally computed sha256 checksums
+sha256  23fc2fec09b621a22699a1fa8685b5b21c64fde7f9385ca6e5f2f29d0a4bc776  LICENSE
--- a/buildroot-external/package/python-srt/python-srt.mk
+++ b/buildroot-external/package/python-srt/python-srt.mk
@ -0,0 +1,14 @@
+################################################################################
+#
+# python-srt
+#
+################################################################################
+
+PYTHON_SRT_VERSION = 3.5.2
+PYTHON_SRT_SOURCE = srt-$(PYTHON_SRT_VERSION).tar.gz
+PYTHON_SRT_SITE = https://files.pythonhosted.org/packages/18/a3/e1466f7c86a9e5d3e462ed6eb3a548917e93cc1ee212cd927f8f4e887ae9
+PYTHON_SRT_SETUP_TYPE = setuptools
+PYTHON_SRT_LICENSE = MIT
+PYTHON_SRT_LICENSE_FILES = LICENSE
+
+$(eval $(python-package))
--- a/buildroot-external/package/python-vosk-api/0001-Minimize-setup.py.patch
+++ b/buildroot-external/package/python-vosk-api/0001-Minimize-setup.py.patch
@ -0,0 +1,82 @@
+From 549711480bb21121a4744da2fecaa50ecc65c619 Mon Sep 17 00:00:00 2001
+From: j1nx <p.steenbergen@j1nx.nl>
+Date: Tue, 20 Dec 2022 14:41:57 +0100
+Subject: [PATCH 1/1] Minimize setup.py
+
+---
+ python/setup.py | 44 ++------------------------------------------
+ 1 file changed, 2 insertions(+), 42 deletions(-)
+
+diff --git a/python/setup.py b/python/setup.py
+index dcfa5de..a1d1c70 100644
+--- a/python/setup.py
+++ b/python/setup.py
+@@ -1,44 +1,5 @@
+ import os
+ import setuptools
+-import shutil
+-import glob
+-import platform
+-
+-# Figure out environment for cross-compile
+-vosk_source = os.getenv("VOSK_SOURCE", os.path.abspath(os.path.join(os.path.dirname(__file__),
+-    "..")))
+-system = os.environ.get('VOSK_SYSTEM', platform.system())
+-architecture = os.environ.get('VOSK_ARCHITECTURE', platform.architecture()[0])
+-machine = os.environ.get('VOSK_MACHINE', platform.machine())
+-
+-# Copy precompmilled libraries
+-for lib in glob.glob(os.path.join(vosk_source, "src/lib*.*")):
+-    print ("Adding library", lib)
+-    shutil.copy(lib, "vosk")
+-
+-# Create OS-dependent, but Python-independent wheels.
+-try:
+-    from wheel.bdist_wheel import bdist_wheel
+-except ImportError:
+-    cmdclass = {}
+-else:
+-    class bdist_wheel_tag_name(bdist_wheel):
+-        def get_tag(self):
+-            abi = 'none'
+-            if system == 'Darwin':
+-                oses = 'macosx_10_6_universal2'
+-            elif system == 'Windows' and architecture == '32bit':
+-                oses = 'win32'
+-            elif system == 'Windows' and architecture == '64bit':
+-                oses = 'win_amd64'
+-            elif system == 'Linux' and machine == 'aarch64' and architecture == '64bit':
+-                oses = 'manylinux2014_aarch64'
+-            elif system == 'Linux':
+-                oses = 'linux_' + machine
+-            else:
+-                raise TypeError("Unknown build environment")
+-            return 'py3', abi, oses
+-    cmdclass = {'bdist_wheel': bdist_wheel_tag_name}
+ 
+ with open("README.md", "rb") as fh:
+     long_description = fh.read().decode("utf-8")
+@@ -52,7 +13,6 @@ setuptools.setup(
+     long_description=long_description,
+     long_description_content_type="text/markdown",
+     url="https://github.com/alphacep/vosk-api",
+-    packages=setuptools.find_packages(),
+     package_data = {'vosk': ['*.so', '*.dll', '*.dyld']},
+     entry_points = {
+         'console_scripts': ['vosk-transcriber=vosk.transcriber.cli:main'],
+@@ -66,10 +26,10 @@ setuptools.setup(
+         'Operating System :: MacOS :: MacOS X',
+         'Topic :: Software Development :: Libraries :: Python Modules'
+     ],
+-    cmdclass=cmdclass,
+    cmdclass={},
+     python_requires='>=3',
+     zip_safe=False, # Since we load so file from the filesystem, we can not run from zip file
+-    setup_requires=['cffi>=1.0', 'requests', 'tqdm', 'srt', 'websockets'],
+    setup_requires=['cffi>=1.0'],
+     install_requires=['cffi>=1.0', 'requests', 'tqdm', 'srt', 'websockets'],
+     cffi_modules=['vosk_builder.py:ffibuilder'],
+ )
+-- 
+2.34.1
+
--- a/buildroot-external/package/python-vosk-api/Config.in
+++ b/buildroot-external/package/python-vosk-api/Config.in
@ -0,0 +1,15 @@
+config BR2_PACKAGE_PYTHON_VOSK_API
+	bool "python-vosk-api"
+	select BR2_PACKAGE_PYTHON_CFFI
+	select BR2_PACKAGE_PYTHON_TQDM
+	select BR2_PACKAGE_PYTHON_REQUESTS
+	select BR2_PACKAGE_PYTHON_SRT
+	select BR2_PACKAGE_PYTHON_WEBSOCKETS
+	select BR2_PACKAGE_VOSK_API
+	help
+	  Offline speech recognition API for Android, iOS, 
+	  Raspberry Pi and servers with Python, Java, C# 
+	  and Node.
+
+	  https://github.com/alphacep/vosk-api.git
+
--- a/buildroot-external/package/python-vosk-api/python-vosk-api.hash
+++ b/buildroot-external/package/python-vosk-api/python-vosk-api.hash
@ -0,0 +1,2 @@
+# Locally computed
+sha256  51ec637e7b3ef42ed3df44530533a26b0798b0f0724e467eab85b6951f51c515  python-vosk-api-cf2560c9f8a49d3d366b433fdabd78c518231bec.tar.gz
--- a/buildroot-external/package/python-vosk-api/python-vosk-api.mk
+++ b/buildroot-external/package/python-vosk-api/python-vosk-api.mk
@ -0,0 +1,23 @@
+################################################################################
+#
+# python-vosk-api
+#
+################################################################################
+
+PYTHON_VOSK_API_VERSION = cf2560c9f8a49d3d366b433fdabd78c518231bec
+PYTHON_VOSK_API_SITE = $(call github,alphacep,vosk-api,$(PYTHON_VOSK_API_VERSION))
+PYTHON_VOSK_API_LICENSE = Apache License 2.0
+
+PYTHON_VOSK_API_DEPENDENCIES = python-cffi python-tqdm python-requests \
+				python-srt python-websockets vosk-api
+PYTHON_VOSK_API_SUBDIR = python
+PYTHON_VOSK_API_SETUP_TYPE = setuptools
+
+PYTHON_VOSK_API_POST_INSTALL_TARGET_HOOKS = PYTHON_VOSK_API_LIBVOSK_SO
+
+define PYTHON_VOSK_API_LIBVOSK_SO
+	$(INSTALL) -D -m 755 $(TARGET_DIR)/usr/lib/libvosk.so \
+	$(TARGET_DIR)/usr/lib/python$(PYTHON3_VERSION_MAJOR)/site-packages/vosk/
+endef
+
+$(eval $(python-package))
--- a/buildroot-external/package/vosk-api/0001-No-kaldi-cmake.patch
+++ b/buildroot-external/package/vosk-api/0001-No-kaldi-cmake.patch
@ -0,0 +1,25 @@
+From f26985185d5186be22c79a27969391b74d0987a0 Mon Sep 17 00:00:00 2001
+From: j1nx <p.steenbergen@j1nx.nl>
+Date: Mon, 19 Dec 2022 09:50:08 +0100
+Subject: [PATCH 1/1] No kaldi cmake
+
+---
+ CMakeLists.txt | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 6101a62..4bdd99b 100644
+--- a/CMakeLists.txt
+++ b/CMakeLists.txt
+@@ -12,7 +12,7 @@ add_library(vosk
+   src/vosk_api.cc
+ )
+ 
+-find_package(kaldi REQUIRED)
+#find_package(kaldi REQUIRED)
+ target_link_libraries(vosk PUBLIC kaldi-base kaldi-online2 kaldi-rnnlm fstngram)
+ 
+ include(GNUInstallDirs)
+-- 
+2.34.1
+
--- a/buildroot-external/package/vosk-api/Config.in
+++ b/buildroot-external/package/vosk-api/Config.in
@ -0,0 +1,10 @@
+config BR2_PACKAGE_VOSK_API
+	bool "vosk-api"
+	select BR2_PACKAGE_KALDI
+	help
+	  Offline speech recognition API for Android, iOS, 
+	  Raspberry Pi and servers with Python, Java, C# 
+	  and Node
+
+	  https://github.com/alphacep/vosk-api.git
+
--- a/buildroot-external/package/vosk-api/vosk-api.hash
+++ b/buildroot-external/package/vosk-api/vosk-api.hash
@ -0,0 +1,2 @@
+# Locally computed
+sha256  51ec637e7b3ef42ed3df44530533a26b0798b0f0724e467eab85b6951f51c515  vosk-api-cf2560c9f8a49d3d366b433fdabd78c518231bec.tar.gz
--- a/buildroot-external/package/vosk-api/vosk-api.mk
+++ b/buildroot-external/package/vosk-api/vosk-api.mk
@ -0,0 +1,20 @@
+################################################################################
+#
+# vosk-api
+#
+################################################################################
+
+VOSK_API_VERSION = cf2560c9f8a49d3d366b433fdabd78c518231bec
+VOSK_API_SITE = $(call github,alphacep,vosk-api,$(VOSK_API_VERSION))
+VOSK_API_LICENSE = Apache License 2.0
+
+VOSK_API_INSTALL_STAGING = YES
+VOSK_API_DEPENDENCIES = host-pkgconf kaldi
+
+VOSK_API_CONF_OPTS += \
+	-DCMAKE_C_FLAGS="$(TARGET_CFLAGS) \
+		-I$(STAGING_DIR)/usr/include/kaldi" \
+	-DCMAKE_CXX_FLAGS="$(TARGET_CXXFLAGS) \
+		-I$(STAGING_DIR)/usr/include/kaldi"
+
+$(eval $(cmake-package))