mirror of
https://github.com/OpenVoiceOS/OpenVoiceOS
synced 2025-01-30 10:25:12 +01:00
2313 lines
85 KiB
Diff
2313 lines
85 KiB
Diff
From 1f7076cb5c3b1ff17e76947c080a6072b13124c5 Mon Sep 17 00:00:00 2001
|
|
From: jarbasal <jarbasai@mailfence.com>
|
|
Date: Thu, 30 Sep 2021 00:21:18 +0100
|
|
Subject: [PATCH 1/5] intent utils
|
|
|
|
---
|
|
ovos_utils/__init__.py | 96 +---
|
|
ovos_utils/bracket_expansion.py | 196 +++++++
|
|
ovos_utils/events.py | 80 ++-
|
|
ovos_utils/file_utils.py | 237 +++++++++
|
|
ovos_utils/intents/__init__.py | 421 +--------------
|
|
ovos_utils/intents/converse.py | 201 +++++++
|
|
.../intents/intent_service_interface.py | 495 ++++++++++++++++++
|
|
7 files changed, 1213 insertions(+), 513 deletions(-)
|
|
create mode 100644 ovos_utils/bracket_expansion.py
|
|
create mode 100644 ovos_utils/file_utils.py
|
|
create mode 100644 ovos_utils/intents/converse.py
|
|
create mode 100644 ovos_utils/intents/intent_service_interface.py
|
|
|
|
diff --git a/ovos_utils/__init__.py b/ovos_utils/__init__.py
|
|
index d8b9134..38c66c7 100644
|
|
--- a/ovos_utils/__init__.py
|
|
+++ b/ovos_utils/__init__.py
|
|
@@ -12,14 +12,14 @@
|
|
#
|
|
from threading import Thread
|
|
from time import sleep
|
|
-import os
|
|
-from os.path import isdir, join, dirname
|
|
+from os.path import isdir, join
|
|
import re
|
|
import datetime
|
|
import kthread
|
|
from ovos_utils.network_utils import *
|
|
from inflection import camelize, titleize, transliterate, parameterize, \
|
|
ordinalize
|
|
+from ovos_utils.file_utils import resolve_ovos_resource_file, resolve_resource_file
|
|
|
|
|
|
def ensure_mycroft_import():
|
|
@@ -35,33 +35,6 @@ def ensure_mycroft_import():
|
|
raise
|
|
|
|
|
|
-def resolve_ovos_resource_file(res_name):
|
|
- """Convert a resource into an absolute filename.
|
|
- used internally for ovos resources
|
|
- """
|
|
- # First look for fully qualified file (e.g. a user setting)
|
|
- if os.path.isfile(res_name):
|
|
- return res_name
|
|
-
|
|
- # now look in bundled ovos resources
|
|
- filename = join(dirname(__file__), "res", res_name)
|
|
- if os.path.isfile(filename):
|
|
- return filename
|
|
-
|
|
- # let's look in ovos_workshop if it's installed
|
|
- try:
|
|
- import ovos_workshop
|
|
- pkg_dir = dirname(ovos_workshop.__file__)
|
|
- filename = join(pkg_dir, "res", res_name)
|
|
- if os.path.isfile(filename):
|
|
- return filename
|
|
- filename = join(pkg_dir, "res", "ui", res_name)
|
|
- if os.path.isfile(filename):
|
|
- return filename
|
|
- except:
|
|
- pass
|
|
- return None # Resource cannot be resolved
|
|
-
|
|
|
|
def get_mycroft_root():
|
|
paths = [
|
|
@@ -75,71 +48,6 @@ def get_mycroft_root():
|
|
return None
|
|
|
|
|
|
-def resolve_resource_file(res_name, root_path=None, config=None):
|
|
- """Convert a resource into an absolute filename.
|
|
-
|
|
- Resource names are in the form: 'filename.ext'
|
|
- or 'path/filename.ext'
|
|
-
|
|
- The system wil look for ~/.mycroft/res_name first, and
|
|
- if not found will look at /opt/mycroft/res_name,
|
|
- then finally it will look for res_name in the 'mycroft/res'
|
|
- folder of the source code package.
|
|
-
|
|
- Example:
|
|
- With mycroft running as the user 'bob', if you called
|
|
- resolve_resource_file('snd/beep.wav')
|
|
- it would return either '/home/bob/.mycroft/snd/beep.wav' or
|
|
- '/opt/mycroft/snd/beep.wav' or '.../mycroft/res/snd/beep.wav',
|
|
- where the '...' is replaced by the path where the package has
|
|
- been installed.
|
|
-
|
|
- Args:
|
|
- res_name (str): a resource path/name
|
|
- config (dict): mycroft.conf, to read data directory from
|
|
- Returns:
|
|
- str: path to resource or None if no resource found
|
|
- """
|
|
- if config is None:
|
|
- from ovos_utils.configuration import read_mycroft_config
|
|
- config = read_mycroft_config()
|
|
-
|
|
- # First look for fully qualified file (e.g. a user setting)
|
|
- if os.path.isfile(res_name):
|
|
- return res_name
|
|
-
|
|
- # Now look for ~/.mycroft/res_name (in user folder)
|
|
- filename = os.path.expanduser("~/.mycroft/" + res_name)
|
|
- if os.path.isfile(filename):
|
|
- return filename
|
|
-
|
|
- # Next look for /opt/mycroft/res/res_name
|
|
- data_dir = os.path.expanduser(config.get('data_dir', "/opt/mycroft"))
|
|
- filename = os.path.expanduser(os.path.join(data_dir, res_name))
|
|
- if os.path.isfile(filename):
|
|
- return filename
|
|
-
|
|
- # look in ovos_utils package itself
|
|
- found = resolve_ovos_resource_file(res_name)
|
|
- if found:
|
|
- return found
|
|
-
|
|
- # Finally look for it in the source package
|
|
- paths = [
|
|
- "/opt/venvs/mycroft-core/lib/python3.7/site-packages/", # mark1/2
|
|
- "/opt/venvs/mycroft-core/lib/python3.4/site-packages/ ", # old mark1 installs
|
|
- "/home/pi/mycroft-core" # picroft
|
|
- ]
|
|
- if root_path:
|
|
- paths += [root_path]
|
|
- for p in paths:
|
|
- filename = os.path.join(p, 'mycroft', 'res', res_name)
|
|
- filename = os.path.abspath(os.path.normpath(filename))
|
|
- if os.path.isfile(filename):
|
|
- return filename
|
|
-
|
|
- return None # Resource cannot be resolved
|
|
-
|
|
|
|
def create_killable_daemon(target, args=(), kwargs=None, autostart=True):
|
|
"""Helper to quickly create and start a thread with daemon = True"""
|
|
diff --git a/ovos_utils/bracket_expansion.py b/ovos_utils/bracket_expansion.py
|
|
new file mode 100644
|
|
index 0000000..1c69a6f
|
|
--- /dev/null
|
|
+++ b/ovos_utils/bracket_expansion.py
|
|
@@ -0,0 +1,196 @@
|
|
+import re
|
|
+
|
|
+
|
|
+def expand_parentheses(sent):
|
|
+ """
|
|
+ ['1', '(', '2', '|', '3, ')'] -> [['1', '2'], ['1', '3']]
|
|
+ For example:
|
|
+ Will it (rain|pour) (today|tomorrow|)?
|
|
+ ---->
|
|
+ Will it rain today?
|
|
+ Will it rain tomorrow?
|
|
+ Will it rain?
|
|
+ Will it pour today?
|
|
+ Will it pour tomorrow?
|
|
+ Will it pour?
|
|
+ Args:
|
|
+ sent (list<str>): List of tokens in sentence
|
|
+ Returns:
|
|
+ list<list<str>>: Multiple possible sentences from original
|
|
+ """
|
|
+ return SentenceTreeParser(sent).expand_parentheses()
|
|
+
|
|
+
|
|
+def expand_options(parentheses_line: str) -> list:
|
|
+ """
|
|
+ Convert 'test (a|b)' -> ['test a', 'test b']
|
|
+ Args:
|
|
+ parentheses_line: Input line to expand
|
|
+ Returns:
|
|
+ List of expanded possibilities
|
|
+ """
|
|
+ # 'a(this|that)b' -> [['a', 'this', 'b'], ['a', 'that', 'b']]
|
|
+ options = expand_parentheses(re.split(r'([(|)])', parentheses_line))
|
|
+ return [re.sub(r'\s+', ' ', ' '.join(i)).strip() for i in options]
|
|
+
|
|
+
|
|
+class Fragment:
|
|
+ """(Abstract) empty sentence fragment"""
|
|
+
|
|
+ def __init__(self, tree):
|
|
+ """
|
|
+ Construct a sentence tree fragment which is merely a wrapper for
|
|
+ a list of Strings
|
|
+ Args:
|
|
+ tree (?): Base tree for the sentence fragment, type depends on
|
|
+ subclass, refer to those subclasses
|
|
+ """
|
|
+ self._tree = tree
|
|
+
|
|
+ def tree(self):
|
|
+ """Return the represented sentence tree as raw data."""
|
|
+ return self._tree
|
|
+
|
|
+ def expand(self):
|
|
+ """
|
|
+ Expanded version of the fragment. In this case an empty sentence.
|
|
+ Returns:
|
|
+ List<List<str>>: A list with an empty sentence (= token/string list)
|
|
+ """
|
|
+ return [[]]
|
|
+
|
|
+ def __str__(self):
|
|
+ return self._tree.__str__()
|
|
+
|
|
+ def __repr__(self):
|
|
+ return self._tree.__repr__()
|
|
+
|
|
+
|
|
+class Word(Fragment):
|
|
+ """
|
|
+ Single word in the sentence tree.
|
|
+ Construct with a string as argument.
|
|
+ """
|
|
+
|
|
+ def expand(self):
|
|
+ """
|
|
+ Creates one sentence that contains exactly that word.
|
|
+ Returns:
|
|
+ List<List<str>>: A list with the given string as sentence
|
|
+ (= token/string list)
|
|
+ """
|
|
+ return [[self._tree]]
|
|
+
|
|
+
|
|
+class Sentence(Fragment):
|
|
+ """
|
|
+ A Sentence made of several concatenations/words.
|
|
+ Construct with a List<Fragment> as argument.
|
|
+ """
|
|
+
|
|
+ def expand(self):
|
|
+ """
|
|
+ Creates a combination of all sub-sentences.
|
|
+ Returns:
|
|
+ List<List<str>>: A list with all subsentence expansions combined in
|
|
+ every possible way
|
|
+ """
|
|
+ old_expanded = [[]]
|
|
+ for sub in self._tree:
|
|
+ sub_expanded = sub.expand()
|
|
+ new_expanded = []
|
|
+ while len(old_expanded) > 0:
|
|
+ sentence = old_expanded.pop()
|
|
+ for new in sub_expanded:
|
|
+ new_expanded.append(sentence + new)
|
|
+ old_expanded = new_expanded
|
|
+ return old_expanded
|
|
+
|
|
+
|
|
+class Options(Fragment):
|
|
+ """
|
|
+ A Combination of possible sub-sentences.
|
|
+ Construct with List<Fragment> as argument.
|
|
+ """
|
|
+
|
|
+ def expand(self):
|
|
+ """
|
|
+ Returns all of its options as seperated sub-sentences.
|
|
+ Returns:
|
|
+ List<List<str>>: A list containing the sentences created by all
|
|
+ expansions of its sub-sentences
|
|
+ """
|
|
+ options = []
|
|
+ for option in self._tree:
|
|
+ options.extend(option.expand())
|
|
+ return options
|
|
+
|
|
+
|
|
+class SentenceTreeParser:
|
|
+ """
|
|
+ Generate sentence token trees from a list of tokens
|
|
+ ['1', '(', '2', '|', '3, ')'] -> [['1', '2'], ['1', '3']]
|
|
+ """
|
|
+
|
|
+ def __init__(self, tokens):
|
|
+ self.tokens = tokens
|
|
+
|
|
+ def _parse(self):
|
|
+ """
|
|
+ Generate sentence token trees
|
|
+ ['1', '(', '2', '|', '3, ')'] -> ['1', ['2', '3']]
|
|
+ """
|
|
+ self._current_position = 0
|
|
+ return self._parse_expr()
|
|
+
|
|
+ def _parse_expr(self):
|
|
+ """
|
|
+ Generate sentence token trees from the current position to
|
|
+ the next closing parentheses / end of the list and return it
|
|
+ ['1', '(', '2', '|', '3, ')'] -> ['1', [['2'], ['3']]]
|
|
+ ['2', '|', '3'] -> [['2'], ['3']]
|
|
+ """
|
|
+ # List of all generated sentences
|
|
+ sentence_list = []
|
|
+ # Currently active sentence
|
|
+ cur_sentence = []
|
|
+ sentence_list.append(Sentence(cur_sentence))
|
|
+ # Determine which form the current expression has
|
|
+ while self._current_position < len(self.tokens):
|
|
+ cur = self.tokens[self._current_position]
|
|
+ self._current_position += 1
|
|
+ if cur == '(':
|
|
+ # Parse the subexpression
|
|
+ subexpr = self._parse_expr()
|
|
+ # Check if the subexpression only has one branch
|
|
+ # -> If so, append "(" and ")" and add it as is
|
|
+ normal_brackets = False
|
|
+ if len(subexpr.tree()) == 1:
|
|
+ normal_brackets = True
|
|
+ cur_sentence.append(Word('('))
|
|
+ # add it to the sentence
|
|
+ cur_sentence.append(subexpr)
|
|
+ if normal_brackets:
|
|
+ cur_sentence.append(Word(')'))
|
|
+ elif cur == '|':
|
|
+ # Begin parsing a new sentence
|
|
+ cur_sentence = []
|
|
+ sentence_list.append(Sentence(cur_sentence))
|
|
+ elif cur == ')':
|
|
+ # End parsing the current subexpression
|
|
+ break
|
|
+ # TODO anything special about {sth}?
|
|
+ else:
|
|
+ cur_sentence.append(Word(cur))
|
|
+ return Options(sentence_list)
|
|
+
|
|
+ def _expand_tree(self, tree):
|
|
+ """
|
|
+ Expand a list of sub sentences to all combinated sentences.
|
|
+ ['1', ['2', '3']] -> [['1', '2'], ['1', '3']]
|
|
+ """
|
|
+ return tree.expand()
|
|
+
|
|
+ def expand_parentheses(self):
|
|
+ tree = self._parse()
|
|
+ return self._expand_tree(tree)
|
|
diff --git a/ovos_utils/events.py b/ovos_utils/events.py
|
|
index fddd2a5..a8b6421 100644
|
|
--- a/ovos_utils/events.py
|
|
+++ b/ovos_utils/events.py
|
|
@@ -1,9 +1,81 @@
|
|
-from ovos_utils.log import LOG
|
|
-from ovos_utils.messagebus import Message, FakeBus
|
|
import time
|
|
from datetime import datetime, timedelta
|
|
from inspect import signature
|
|
|
|
+from ovos_utils.intents.intent_service_interface import to_alnum
|
|
+from ovos_utils.log import LOG
|
|
+from ovos_utils.messagebus import Message, FakeBus
|
|
+
|
|
+
|
|
+def unmunge_message(message, skill_id):
|
|
+ """Restore message keywords by removing the Letterified skill ID.
|
|
+ Args:
|
|
+ message (Message): Intent result message
|
|
+ skill_id (str): skill identifier
|
|
+ Returns:
|
|
+ Message without clear keywords
|
|
+ """
|
|
+ if isinstance(message, Message) and isinstance(message.data, dict):
|
|
+ skill_id = to_alnum(skill_id)
|
|
+ for key in list(message.data.keys()):
|
|
+ if key.startswith(skill_id):
|
|
+ # replace the munged key with the real one
|
|
+ new_key = key[len(skill_id):]
|
|
+ message.data[new_key] = message.data.pop(key)
|
|
+
|
|
+ return message
|
|
+
|
|
+
|
|
+def get_handler_name(handler):
|
|
+ """Name (including class if available) of handler function.
|
|
+
|
|
+ Args:
|
|
+ handler (function): Function to be named
|
|
+
|
|
+ Returns:
|
|
+ string: handler name as string
|
|
+ """
|
|
+ if '__self__' in dir(handler) and 'name' in dir(handler.__self__):
|
|
+ return handler.__self__.name + '.' + handler.__name__
|
|
+ else:
|
|
+ return handler.__name__
|
|
+
|
|
+
|
|
+def create_wrapper(handler, skill_id, on_start, on_end, on_error):
|
|
+ """Create the default skill handler wrapper.
|
|
+
|
|
+ This wrapper handles things like metrics, reporting handler start/stop
|
|
+ and errors.
|
|
+ handler (callable): method/function to call
|
|
+ skill_id: skill_id for associated skill
|
|
+ on_start (function): function to call before executing the handler
|
|
+ on_end (function): function to call after executing the handler
|
|
+ on_error (function): function to call for error reporting
|
|
+ """
|
|
+
|
|
+ def wrapper(message):
|
|
+ try:
|
|
+ message = unmunge_message(message, skill_id)
|
|
+ if on_start:
|
|
+ on_start(message)
|
|
+
|
|
+ if len(signature(handler).parameters) == 0:
|
|
+ handler()
|
|
+ else:
|
|
+ handler(message)
|
|
+
|
|
+ except Exception as e:
|
|
+ if on_error:
|
|
+ if len(signature(on_error).parameters) == 2:
|
|
+ on_error(e, message)
|
|
+ else:
|
|
+ on_error(e)
|
|
+ finally:
|
|
+ if on_end:
|
|
+ on_end(message)
|
|
+
|
|
+ return wrapper
|
|
+
|
|
|
|
def create_basic_wrapper(handler, on_error=None):
|
|
"""Create the default skill handler wrapper.
|
|
@@ -18,6 +90,7 @@ def create_basic_wrapper(handler, on_error=None):
|
|
Returns:
|
|
Wrapped callable
|
|
"""
|
|
+
|
|
def wrapper(message):
|
|
try:
|
|
if len(signature(handler).parameters) == 0:
|
|
@@ -37,6 +110,7 @@ class EventContainer:
|
|
This container tracks events added by a skill, allowing unregistering
|
|
all events on shutdown.
|
|
"""
|
|
+
|
|
def __init__(self, bus=None):
|
|
self.bus = bus or FakeBus()
|
|
self.events = []
|
|
@@ -53,6 +127,7 @@ def add(self, name, handler, once=False):
|
|
once (bool, optional): Event handler will be removed after it has
|
|
been run once.
|
|
"""
|
|
+
|
|
def once_wrapper(message):
|
|
# Remove registered one-time handler before invoking,
|
|
# allowing them to re-schedule themselves.
|
|
@@ -113,6 +188,7 @@ def clear(self):
|
|
|
|
class EventSchedulerInterface:
|
|
"""Interface for accessing the event scheduler over the message bus."""
|
|
+
|
|
def __init__(self, name, sched_id=None, bus=None):
|
|
self.name = name
|
|
self.sched_id = sched_id
|
|
diff --git a/ovos_utils/file_utils.py b/ovos_utils/file_utils.py
|
|
new file mode 100644
|
|
index 0000000..de9122a
|
|
--- /dev/null
|
|
+++ b/ovos_utils/file_utils.py
|
|
@@ -0,0 +1,237 @@
|
|
+import collections
|
|
+import csv
|
|
+import re
|
|
+import os
|
|
+from os import walk
|
|
+from os.path import splitext, join, dirname
|
|
+
|
|
+from ovos_utils.bracket_expansion import expand_options
|
|
+from ovos_utils.intents.intent_service_interface import to_alnum, munge_regex
|
|
+from ovos_utils.log import LOG
|
|
+
|
|
+
|
|
+def resolve_ovos_resource_file(res_name):
|
|
+ """Convert a resource into an absolute filename.
|
|
+ used internally for ovos resources
|
|
+ """
|
|
+ # First look for fully qualified file (e.g. a user setting)
|
|
+ if os.path.isfile(res_name):
|
|
+ return res_name
|
|
+
|
|
+ # now look in bundled ovos resources
|
|
+ filename = join(dirname(__file__), "res", res_name)
|
|
+ if os.path.isfile(filename):
|
|
+ return filename
|
|
+
|
|
+ # let's look in ovos_workshop if it's installed
|
|
+ try:
|
|
+ import ovos_workshop
|
|
+ pkg_dir = dirname(ovos_workshop.__file__)
|
|
+ filename = join(pkg_dir, "res", res_name)
|
|
+ if os.path.isfile(filename):
|
|
+ return filename
|
|
+ filename = join(pkg_dir, "res", "ui", res_name)
|
|
+ if os.path.isfile(filename):
|
|
+ return filename
|
|
+ except:
|
|
+ pass
|
|
+ return None # Resource cannot be resolved
|
|
+
|
|
+
|
|
+def resolve_resource_file(res_name, root_path=None, config=None):
|
|
+ """Convert a resource into an absolute filename.
|
|
+
|
|
+ Resource names are in the form: 'filename.ext'
|
|
+ or 'path/filename.ext'
|
|
+
|
|
+ The system wil look for ~/.mycroft/res_name first, and
|
|
+ if not found will look at /opt/mycroft/res_name,
|
|
+ then finally it will look for res_name in the 'mycroft/res'
|
|
+ folder of the source code package.
|
|
+
|
|
+ Example:
|
|
+ With mycroft running as the user 'bob', if you called
|
|
+ resolve_resource_file('snd/beep.wav')
|
|
+ it would return either '/home/bob/.mycroft/snd/beep.wav' or
|
|
+ '/opt/mycroft/snd/beep.wav' or '.../mycroft/res/snd/beep.wav',
|
|
+ where the '...' is replaced by the path where the package has
|
|
+ been installed.
|
|
+
|
|
+ Args:
|
|
+ res_name (str): a resource path/name
|
|
+ config (dict): mycroft.conf, to read data directory from
|
|
+ Returns:
|
|
+ str: path to resource or None if no resource found
|
|
+ """
|
|
+ if config is None:
|
|
+ from ovos_utils.configuration import read_mycroft_config
|
|
+ config = read_mycroft_config()
|
|
+
|
|
+ # First look for fully qualified file (e.g. a user setting)
|
|
+ if os.path.isfile(res_name):
|
|
+ return res_name
|
|
+
|
|
+ # Now look for ~/.mycroft/res_name (in user folder)
|
|
+ filename = os.path.expanduser("~/.mycroft/" + res_name)
|
|
+ if os.path.isfile(filename):
|
|
+ return filename
|
|
+
|
|
+ # Next look for /opt/mycroft/res/res_name
|
|
+ data_dir = os.path.expanduser(config.get('data_dir', "/opt/mycroft"))
|
|
+ filename = os.path.expanduser(os.path.join(data_dir, res_name))
|
|
+ if os.path.isfile(filename):
|
|
+ return filename
|
|
+
|
|
+ # look in ovos_utils package itself
|
|
+ found = resolve_ovos_resource_file(res_name)
|
|
+ if found:
|
|
+ return found
|
|
+
|
|
+ # Finally look for it in the source package
|
|
+ paths = [
|
|
+ "/opt/venvs/mycroft-core/lib/python3.7/site-packages/", # mark1/2
|
|
+ "/opt/venvs/mycroft-core/lib/python3.4/site-packages/ ",
|
|
+ # old mark1 installs
|
|
+ "/home/pi/mycroft-core" # picroft
|
|
+ ]
|
|
+ if root_path:
|
|
+ paths += [root_path]
|
|
+ for p in paths:
|
|
+ filename = os.path.join(p, 'mycroft', 'res', res_name)
|
|
+ filename = os.path.abspath(os.path.normpath(filename))
|
|
+ if os.path.isfile(filename):
|
|
+ return filename
|
|
+
|
|
+ return None # Resource cannot be resolved
|
|
+
|
|
+
|
|
+def read_vocab_file(path):
|
|
+ """ Read voc file.
|
|
+
|
|
+ This reads a .voc file, stripping out empty lines comments and expand
|
|
+ parentheses. It returns each line as a list of all expanded
|
|
+ alternatives.
|
|
+
|
|
+ Args:
|
|
+ path (str): path to vocab file.
|
|
+
|
|
+ Returns:
|
|
+ List of Lists of strings.
|
|
+ """
|
|
+ vocab = []
|
|
+ with open(path, 'r', encoding='utf8') as voc_file:
|
|
+ for line in voc_file.readlines():
|
|
+ if line.startswith('#') or line.strip() == '':
|
|
+ continue
|
|
+ vocab.append(expand_options(line.lower()))
|
|
+ return vocab
|
|
+
|
|
+
|
|
+def load_regex_from_file(path, skill_id):
|
|
+ """Load regex from file
|
|
+ The regex is sent to the intent handler using the message bus
|
|
+
|
|
+ Args:
|
|
+ path: path to vocabulary file (*.voc)
|
|
+ skill_id: skill_id to the regex is tied to
|
|
+ """
|
|
+ regexes = []
|
|
+ if path.endswith('.rx'):
|
|
+ with open(path, 'r', encoding='utf8') as reg_file:
|
|
+ for line in reg_file.readlines():
|
|
+ if line.startswith("#"):
|
|
+ continue
|
|
+ LOG.debug('regex pre-munge: ' + line.strip())
|
|
+ regex = munge_regex(line.strip(), skill_id)
|
|
+ LOG.debug('regex post-munge: ' + regex)
|
|
+ # Raise error if regex can't be compiled
|
|
+ try:
|
|
+ re.compile(regex)
|
|
+ regexes.append(regex)
|
|
+ except Exception as e:
|
|
+ LOG.warning(f'Failed to compile regex {regex}: {e}')
|
|
+
|
|
+ return regexes
|
|
+
|
|
+
|
|
+def load_vocabulary(basedir, skill_id):
|
|
+ """Load vocabulary from all files in the specified directory.
|
|
+
|
|
+ Args:
|
|
+ basedir (str): path of directory to load from (will recurse)
|
|
+ skill_id: skill the data belongs to
|
|
+ Returns:
|
|
+ dict with intent_type as keys and list of list of lists as value.
|
|
+ """
|
|
+ vocabs = {}
|
|
+ for path, _, files in walk(basedir):
|
|
+ for f in files:
|
|
+ if f.endswith(".voc"):
|
|
+ vocab_type = to_alnum(skill_id) + splitext(f)[0]
|
|
+ vocs = read_vocab_file(join(path, f))
|
|
+ if vocs:
|
|
+ vocabs[vocab_type] = vocs
|
|
+ return vocabs
|
|
+
|
|
+
|
|
+def load_regex(basedir, skill_id):
|
|
+ """Load regex from all files in the specified directory.
|
|
+
|
|
+ Args:
|
|
+ basedir (str): path of directory to load from
|
|
+ bus (messagebus emitter): messagebus instance used to send the vocab to
|
|
+ the intent service
|
|
+ skill_id (str): skill identifier
|
|
+ """
|
|
+ regexes = []
|
|
+ for path, _, files in walk(basedir):
|
|
+ for f in files:
|
|
+ if f.endswith(".rx"):
|
|
+ regexes += load_regex_from_file(join(path, f), skill_id)
|
|
+ return regexes
|
|
+
|
|
+
|
|
+def read_value_file(filename, delim):
|
|
+ """Read value file.
|
|
+
|
|
+ The value file is a simple csv structure with a key and value.
|
|
+
|
|
+ Args:
|
|
+ filename (str): file to read
|
|
+ delim (str): csv delimiter
|
|
+
|
|
+ Returns:
|
|
+ OrderedDict with results.
|
|
+ """
|
|
+ result = collections.OrderedDict()
|
|
+
|
|
+ if filename:
|
|
+ with open(filename) as f:
|
|
+ reader = csv.reader(f, delimiter=delim)
|
|
+ for row in reader:
|
|
+ # skip blank or comment lines
|
|
+ if not row or row[0].startswith("#"):
|
|
+ continue
|
|
+ if len(row) != 2:
|
|
+ continue
|
|
+
|
|
+ result[row[0]] = row[1]
|
|
+ return result
|
|
+
|
|
+
|
|
+def read_translated_file(filename, data):
|
|
+ """Read a file inserting data.
|
|
+
|
|
+ Args:
|
|
+ filename (str): file to read
|
|
+ data (dict): dictionary with data to insert into file
|
|
+
|
|
+ Returns:
|
|
+ list of lines.
|
|
+ """
|
|
+ if filename:
|
|
+ with open(filename) as f:
|
|
+ text = f.read().replace('{{', '{').replace('}}', '}')
|
|
+ return text.format(**data or {}).rstrip('\n').split('\n')
|
|
+ else:
|
|
+ return None
|
|
diff --git a/ovos_utils/intents/__init__.py b/ovos_utils/intents/__init__.py
|
|
index 6165119..ff9e985 100644
|
|
--- a/ovos_utils/intents/__init__.py
|
|
+++ b/ovos_utils/intents/__init__.py
|
|
@@ -1,417 +1,4 @@
|
|
-from ovos_utils.log import LOG
|
|
-from ovos_utils.messagebus import get_mycroft_bus, Message
|
|
-import time
|
|
-from os.path import isfile
|
|
-
|
|
-
|
|
-class IntentQueryApi:
|
|
- """
|
|
- Query Intent Service at runtime
|
|
- """
|
|
-
|
|
- def __init__(self, bus=None, timeout=5):
|
|
- self.bus = bus or get_mycroft_bus()
|
|
- self.timeout = timeout
|
|
-
|
|
- def get_adapt_intent(self, utterance, lang="en-us"):
|
|
- """ get best adapt intent for utterance """
|
|
- msg = Message("intent.service.adapt.get",
|
|
- {"utterance": utterance, "lang": lang},
|
|
- context={"destination": "intent_service",
|
|
- "source": "intent_api"})
|
|
-
|
|
- resp = self.bus.wait_for_response(msg,
|
|
- 'intent.service.adapt.reply',
|
|
- timeout=self.timeout)
|
|
- data = resp.data if resp is not None else {}
|
|
- if not data:
|
|
- LOG.error("Intent Service timed out!")
|
|
- return None
|
|
- return data["intent"]
|
|
-
|
|
- def get_padatious_intent(self, utterance, lang="en-us"):
|
|
- """ get best padatious intent for utterance """
|
|
- msg = Message("intent.service.padatious.get",
|
|
- {"utterance": utterance, "lang": lang},
|
|
- context={"destination": "intent_service",
|
|
- "source": "intent_api"})
|
|
- resp = self.bus.wait_for_response(msg,
|
|
- 'intent.service.padatious.reply',
|
|
- timeout=self.timeout)
|
|
- data = resp.data if resp is not None else {}
|
|
- if not data:
|
|
- LOG.error("Intent Service timed out!")
|
|
- return None
|
|
- return data["intent"]
|
|
-
|
|
- def get_intent(self, utterance, lang="en-us"):
|
|
- """ get best intent for utterance """
|
|
- msg = Message("intent.service.intent.get",
|
|
- {"utterance": utterance, "lang": lang},
|
|
- context={"destination": "intent_service",
|
|
- "source": "intent_api"})
|
|
- resp = self.bus.wait_for_response(msg,
|
|
- 'intent.service.intent.reply',
|
|
- timeout=self.timeout)
|
|
- data = resp.data if resp is not None else {}
|
|
- if not data:
|
|
- LOG.error("Intent Service timed out!")
|
|
- return None
|
|
- return data["intent"]
|
|
-
|
|
- def get_skill(self, utterance, lang="en-us"):
|
|
- """ get skill that utterance will trigger """
|
|
- intent = self.get_intent(utterance, lang)
|
|
- if not intent:
|
|
- return None
|
|
- # theoretically skill_id might be missing
|
|
- if intent.get("skill_id"):
|
|
- return intent["skill_id"]
|
|
- # retrieve skill from munged intent name
|
|
- if intent.get("intent_name"): # padatious + adapt
|
|
- return intent["name"].split(":")[0]
|
|
- if intent.get("intent_type"): # adapt
|
|
- return intent["intent_type"].split(":")[0]
|
|
- return None # raise some error here maybe? this should never happen
|
|
-
|
|
- def get_skills_manifest(self):
|
|
- msg = Message("intent.service.skills.get",
|
|
- context={"destination": "intent_service",
|
|
- "source": "intent_api"})
|
|
- resp = self.bus.wait_for_response(msg,
|
|
- 'intent.service.skills.reply',
|
|
- timeout=self.timeout)
|
|
- data = resp.data if resp is not None else {}
|
|
- if not data:
|
|
- LOG.error("Intent Service timed out!")
|
|
- return None
|
|
- return data["skills"]
|
|
-
|
|
- def get_active_skills(self, include_timestamps=False):
|
|
- msg = Message("intent.service.active_skills.get",
|
|
- context={"destination": "intent_service",
|
|
- "source": "intent_api"})
|
|
- resp = self.bus.wait_for_response(msg,
|
|
- 'intent.service.active_skills.reply',
|
|
- timeout=self.timeout)
|
|
- data = resp.data if resp is not None else {}
|
|
- if not data:
|
|
- LOG.error("Intent Service timed out!")
|
|
- return None
|
|
- if include_timestamps:
|
|
- return data["skills"]
|
|
- # HACK waiting for https://github.com/MycroftAI/mycroft-core/pull/2786
|
|
- if len(data["skills"]) and not len(data["skills"][0]) == 2:
|
|
- return data["skills"]
|
|
- return [s[0] for s in data["skills"]]
|
|
-
|
|
- def get_adapt_manifest(self):
|
|
- msg = Message("intent.service.adapt.manifest.get",
|
|
- context={"destination": "intent_service",
|
|
- "source": "intent_api"})
|
|
- resp = self.bus.wait_for_response(msg,
|
|
- 'intent.service.adapt.manifest',
|
|
- timeout=self.timeout)
|
|
- data = resp.data if resp is not None else {}
|
|
- if not data:
|
|
- LOG.error("Intent Service timed out!")
|
|
- return None
|
|
- return data["intents"]
|
|
-
|
|
- def get_padatious_manifest(self):
|
|
- msg = Message("intent.service.padatious.manifest.get",
|
|
- context={"destination": "intent_service",
|
|
- "source": "intent_api"})
|
|
- resp = self.bus.wait_for_response(msg,
|
|
- 'intent.service.padatious.manifest',
|
|
- timeout=self.timeout)
|
|
- data = resp.data if resp is not None else {}
|
|
- if not data:
|
|
- LOG.error("Intent Service timed out!")
|
|
- return None
|
|
- return data["intents"]
|
|
-
|
|
- def get_intent_manifest(self):
|
|
- padatious = self.get_padatious_manifest()
|
|
- adapt = self.get_adapt_manifest()
|
|
- return {"adapt": adapt,
|
|
- "padatious": padatious}
|
|
-
|
|
- def get_vocab_manifest(self):
|
|
- msg = Message("intent.service.adapt.vocab.manifest.get",
|
|
- context={"destination": "intent_service",
|
|
- "source": "intent_api"})
|
|
- reply_msg_type = 'intent.service.adapt.vocab.manifest'
|
|
- resp = self.bus.wait_for_response(msg,
|
|
- reply_msg_type,
|
|
- timeout=self.timeout)
|
|
- data = resp.data if resp is not None else {}
|
|
- if not data:
|
|
- LOG.error("Intent Service timed out!")
|
|
- return None
|
|
-
|
|
- vocab = {}
|
|
- for voc in data["vocab"]:
|
|
- if voc.get("regex"):
|
|
- continue
|
|
- if voc["end"] not in vocab:
|
|
- vocab[voc["end"]] = {"samples": []}
|
|
- vocab[voc["end"]]["samples"].append(voc["start"])
|
|
- return [{"name": voc, "samples": vocab[voc]["samples"]}
|
|
- for voc in vocab]
|
|
-
|
|
- def get_regex_manifest(self):
|
|
- msg = Message("intent.service.adapt.vocab.manifest.get",
|
|
- context={"destination": "intent_service",
|
|
- "source": "intent_api"})
|
|
- reply_msg_type = 'intent.service.adapt.vocab.manifest'
|
|
- resp = self.bus.wait_for_response(msg,
|
|
- reply_msg_type,
|
|
- timeout=self.timeout)
|
|
- data = resp.data if resp is not None else {}
|
|
- if not data:
|
|
- LOG.error("Intent Service timed out!")
|
|
- return None
|
|
-
|
|
- vocab = {}
|
|
- for voc in data["vocab"]:
|
|
- if not voc.get("regex"):
|
|
- continue
|
|
- name = voc["regex"].split("(?P<")[-1].split(">")[0]
|
|
- if name not in vocab:
|
|
- vocab[name] = {"samples": []}
|
|
- vocab[name]["samples"].append(voc["regex"])
|
|
- return [{"name": voc, "regexes": vocab[voc]["samples"]}
|
|
- for voc in vocab]
|
|
-
|
|
- def get_entities_manifest(self):
|
|
- msg = Message("intent.service.padatious.entities.manifest.get",
|
|
- context={"destination": "intent_service",
|
|
- "source": "intent_api"})
|
|
- reply_msg_type = 'intent.service.padatious.entities.manifest'
|
|
- resp = self.bus.wait_for_response(msg,
|
|
- reply_msg_type,
|
|
- timeout=self.timeout)
|
|
- data = resp.data if resp is not None else {}
|
|
- if not data:
|
|
- LOG.error("Intent Service timed out!")
|
|
- return None
|
|
-
|
|
- entities = []
|
|
- # read files
|
|
- for ent in data["entities"]:
|
|
- if isfile(ent["file_name"]):
|
|
- with open(ent["file_name"]) as f:
|
|
- lines = f.read().replace("(", "").replace(")", "").split(
|
|
- "\n")
|
|
- samples = []
|
|
- for l in lines:
|
|
- samples += [a.strip() for a in l.split("|") if a.strip()]
|
|
- entities.append({"name": ent["name"], "samples": samples})
|
|
- return entities
|
|
-
|
|
- def get_keywords_manifest(self):
|
|
- padatious = self.get_entities_manifest()
|
|
- adapt = self.get_vocab_manifest()
|
|
- regex = self.get_regex_manifest()
|
|
- return {"adapt": adapt,
|
|
- "padatious": padatious,
|
|
- "regex": regex}
|
|
-
|
|
-
|
|
-class ConverseTracker:
|
|
- """ Using the messagebus this class recreates/keeps track of the state
|
|
- of the converse system, it uses both passive listening and active
|
|
- queries to sync it's state, it also emits 2 new bus events
|
|
-
|
|
- Implements https://github.com/MycroftAI/mycroft-core/pull/1468
|
|
- """
|
|
- bus = None
|
|
- active_skills = []
|
|
- converse_timeout = 5 # MAGIC NUMBER hard coded in mycroft-core
|
|
- last_conversed = None
|
|
- intent_api = None
|
|
-
|
|
- @classmethod
|
|
- def connect_bus(cls, mycroft_bus):
|
|
- """Registers the bus object to use."""
|
|
- # PATCH - in mycroft-core this would be handled in intent_service
|
|
- # in here it is done in MycroftSkill.bind so i added this
|
|
- # conditional check
|
|
- if cls.bus is None and mycroft_bus is not None:
|
|
- cls.bus = mycroft_bus
|
|
- cls.intent_api = IntentQueryApi(cls.bus)
|
|
- cls.register_bus_events()
|
|
-
|
|
- @classmethod
|
|
- def register_bus_events(cls):
|
|
- cls.bus.on('active_skill_request', cls.handle_activate_request)
|
|
- cls.bus.on('skill.converse.response', cls.handle_converse_response)
|
|
- cls.bus.on("mycroft.skill.handler.start", cls.handle_intent_start)
|
|
- cls.bus.on("recognizer_loop:utterance", cls.handle_utterance)
|
|
-
|
|
- # public methods
|
|
- @classmethod
|
|
- def check_skill(cls, skill_id):
|
|
- """ Check if a skill is active """
|
|
- cls.filter_active_skills()
|
|
- for skill in list(cls.active_skills):
|
|
- if skill[0] == skill_id:
|
|
- return True
|
|
- return False
|
|
-
|
|
- @classmethod
|
|
- def filter_active_skills(cls):
|
|
- """ Removes expired skills from active skill list """
|
|
- # filter timestamps
|
|
- for skill in list(cls.active_skills):
|
|
- if time.time() - skill[1] <= cls.converse_timeout * 60:
|
|
- cls.remove_active_skill(skill[0])
|
|
-
|
|
- @classmethod
|
|
- def sync_with_intent_service(cls):
|
|
- """sync active skill list using intent api
|
|
-
|
|
- WARNING
|
|
- we don't have the timestamps so order might be messed up!!
|
|
- avoid calling this until
|
|
- """
|
|
- skill_ids = cls.intent_api.get_active_skills(include_timestamps=True)
|
|
- if skill_ids:
|
|
- if len(skill_ids[0]) == 2:
|
|
- # PR was merged! hurray!
|
|
- cls.active_skills = skill_ids
|
|
- else:
|
|
- # hoping they come sorted by timestamp....
|
|
- # older to newer (most recently used)
|
|
- for skill_id in reversed(skill_ids):
|
|
- # are we tracking this skill ?
|
|
- if not cls.check_skill(skill_id):
|
|
- # we missed adding this skill in our tracking
|
|
- cls.add_active_skill(skill_id)
|
|
- for skill in cls.active_skills:
|
|
- if skill[0] not in skill_ids:
|
|
- # we missed removing this skill in our tracking
|
|
- cls.remove_active_skill(skill[0])
|
|
-
|
|
- # https://github.com/MycroftAI/mycroft-core/pull/1468
|
|
- @classmethod
|
|
- def remove_active_skill(cls, skill_id, silent=False):
|
|
- """
|
|
- Emits "converse.skill.deactivated" event, improvement of #1468
|
|
- """
|
|
- for skill in list(cls.active_skills):
|
|
- if skill[0] == skill_id:
|
|
- cls.active_skills.remove(skill)
|
|
- if not silent:
|
|
- cls.bus.emit(Message("converse.skill.deactivated",
|
|
- {"skill_id": skill[0]}))
|
|
-
|
|
- @classmethod
|
|
- def add_active_skill(cls, skill_id):
|
|
- """
|
|
- Emits "converse.skill.activated" event, improvement of #1468
|
|
- """
|
|
- # search the list for an existing entry that already contains it
|
|
- # and remove that reference
|
|
- if skill_id != '':
|
|
- cls.remove_active_skill(skill_id, silent=True)
|
|
- # add skill with timestamp to start of skill_list
|
|
- cls.active_skills.insert(0, [skill_id, time.time()])
|
|
- # this might be sent more than once and it's perfectly fine
|
|
- # it's just a new info message not consumed anywhere by default
|
|
- cls.bus.emit(Message("converse.skill.activated",
|
|
- {"skill_id": skill_id}))
|
|
- else:
|
|
- LOG.warning('Skill ID was empty, won\'t add to list of '
|
|
- 'active skills.')
|
|
-
|
|
- # status tracking
|
|
- @classmethod
|
|
- def handle_activate_request(cls, message):
|
|
- """
|
|
- a skill bumped itself to the top of active skills list
|
|
- duplicate functionality from mycroft-core, keeping list in sync
|
|
- """
|
|
- skill_id = message.data["skill_id"]
|
|
- cls.add_active_skill(skill_id)
|
|
-
|
|
- @classmethod
|
|
- def handle_converse_error(cls, message):
|
|
- """
|
|
- a skill was removed from active skill list due to converse error
|
|
- duplicate functionality from mycroft-core, keeping list in sync
|
|
- """
|
|
- skill_id = message.data["skill_id"]
|
|
- if message.data["error"] == "skill id does not exist":
|
|
- cls.remove_active_skill(skill_id)
|
|
-
|
|
- @classmethod
|
|
- def handle_intent_start(cls, message):
|
|
- """
|
|
- duplicate functionality from mycroft-core, keeping list in sync
|
|
-
|
|
- TODO skill_id from message, core is not passing it along... it used
|
|
- to be possible to retrieve it from munged message but that changed.
|
|
- send a PR (if those got merged this code wouldn't exist)
|
|
-
|
|
- handle_utterance will take over this functionality for now
|
|
- handle_converse_response will take corrective action
|
|
- """
|
|
- # skill_id = message.data["skill_id"]
|
|
- # bump skill to top of active list
|
|
- # cls.add_active_skill(skill_id)
|
|
-
|
|
- @classmethod
|
|
- def handle_utterance(cls, message):
|
|
- """
|
|
- duplicate functionality from mycroft-core, keeping list in sync
|
|
-
|
|
- WORKAROUND - skill_id missing in handle_intent_start, will keep list
|
|
- in sync by using the IntentAPI to check what skill the utterance
|
|
- should trigger
|
|
-
|
|
- handle_converse_response will take corrective action
|
|
- """
|
|
- # NOTE borked in mycroft-core
|
|
- # needs https://github.com/MycroftAI/mycroft-core/pull/2786
|
|
- skill_id = cls.intent_api.get_skill(message.data["utterances"][0])
|
|
- if skill_id:
|
|
- # this skill will trigger and therefore is the last active skill
|
|
- cls.add_active_skill(skill_id)
|
|
- # will remove expired intents from list
|
|
- cls.filter_active_skills()
|
|
-
|
|
- @classmethod
|
|
- def handle_converse_response(cls, message):
|
|
- """
|
|
- tracks last_conversed skill
|
|
-
|
|
- FAILSAFE - additional checks to correct active skills list,
|
|
- but that should never happen, accounts for mistakes in
|
|
- handle_utterance / intent_api
|
|
-
|
|
- accounts for https://github.com/MycroftAI/mycroft-core/pull/2786
|
|
- not yet being merged
|
|
- """
|
|
- skill_id = message.data["skill_id"]
|
|
- if 'error' in message.data:
|
|
- cls.handle_converse_error(message)
|
|
- elif message.data.get('result') is True:
|
|
- cls.last_conversed = skill_id
|
|
- if not cls.check_skill(skill_id):
|
|
- # seems like we missed adding this skill to active list
|
|
- # NOTE this is a failsafe and should never trigger
|
|
- # since this answered True we have the real timestamp
|
|
- cls.add_active_skill(skill_id)
|
|
- elif not cls.check_skill(skill_id):
|
|
- # seems like we missed adding this skill to active list
|
|
- # NOTE this is a failsafe and should never trigger
|
|
- # since this answered false and we don't have the real timestamp
|
|
- # let's add it to the end of the active_skills list
|
|
- ts = time.time()
|
|
- if len(cls.active_skills):
|
|
- ts = cls.active_skills[-1][1]
|
|
- cls.active_skills.append([skill_id, ts])
|
|
-
|
|
-
|
|
+from ovos_utils.intents.intent_service_interface import IntentQueryApi, \
|
|
+ IntentServiceInterface
|
|
+from ovos_utils.intents.converse import ConverseTracker
|
|
+from ovos_utils.intents.layers import IntentLayers
|
|
diff --git a/ovos_utils/intents/converse.py b/ovos_utils/intents/converse.py
|
|
new file mode 100644
|
|
index 0000000..7970585
|
|
--- /dev/null
|
|
+++ b/ovos_utils/intents/converse.py
|
|
@@ -0,0 +1,201 @@
|
|
+import time
|
|
+
|
|
+from ovos_utils.intents.intent_service_interface import IntentQueryApi
|
|
+from ovos_utils.log import LOG
|
|
+from ovos_utils.messagebus import Message
|
|
+
|
|
+
|
|
+class ConverseTracker:
|
|
+ """ Using the messagebus this class recreates/keeps track of the state
|
|
+ of the converse system, it uses both passive listening and active
|
|
+ queries to sync it's state, it also emits 2 new bus events
|
|
+
|
|
+ Implements https://github.com/MycroftAI/mycroft-core/pull/1468
|
|
+ """
|
|
+ bus = None
|
|
+ active_skills = []
|
|
+ converse_timeout = 5 # MAGIC NUMBER hard coded in mycroft-core
|
|
+ last_conversed = None
|
|
+ intent_api = None
|
|
+
|
|
+ @classmethod
|
|
+ def connect_bus(cls, mycroft_bus):
|
|
+ """Registers the bus object to use."""
|
|
+ # PATCH - in mycroft-core this would be handled in intent_service
|
|
+ # in here it is done in MycroftSkill.bind so i added this
|
|
+ # conditional check
|
|
+ if cls.bus is None and mycroft_bus is not None:
|
|
+ cls.bus = mycroft_bus
|
|
+ cls.intent_api = IntentQueryApi(cls.bus)
|
|
+ cls.register_bus_events()
|
|
+
|
|
+ @classmethod
|
|
+ def register_bus_events(cls):
|
|
+ cls.bus.on('active_skill_request', cls.handle_activate_request)
|
|
+ cls.bus.on('skill.converse.response', cls.handle_converse_response)
|
|
+ cls.bus.on("mycroft.skill.handler.start", cls.handle_intent_start)
|
|
+ cls.bus.on("recognizer_loop:utterance", cls.handle_utterance)
|
|
+
|
|
+ # public methods
|
|
+ @classmethod
|
|
+ def check_skill(cls, skill_id):
|
|
+ """ Check if a skill is active """
|
|
+ cls.filter_active_skills()
|
|
+ for skill in list(cls.active_skills):
|
|
+ if skill[0] == skill_id:
|
|
+ return True
|
|
+ return False
|
|
+
|
|
+ @classmethod
|
|
+ def filter_active_skills(cls):
|
|
+ """ Removes expired skills from active skill list """
|
|
+ # filter timestamps
|
|
+ for skill in list(cls.active_skills):
|
|
+ if time.time() - skill[1] <= cls.converse_timeout * 60:
|
|
+ cls.remove_active_skill(skill[0])
|
|
+
|
|
+ @classmethod
|
|
+ def sync_with_intent_service(cls):
|
|
+ """sync active skill list using intent api
|
|
+
|
|
+ WARNING
|
|
+ we don't have the timestamps so order might be messed up!!
|
|
+ avoid calling this until
|
|
+ """
|
|
+ skill_ids = cls.intent_api.get_active_skills(include_timestamps=True)
|
|
+ if skill_ids:
|
|
+ if len(skill_ids[0]) == 2:
|
|
+ # PR was merged! hurray!
|
|
+ cls.active_skills = skill_ids
|
|
+ else:
|
|
+ # hoping they come sorted by timestamp....
|
|
+ # older to newer (most recently used)
|
|
+ for skill_id in reversed(skill_ids):
|
|
+ # are we tracking this skill ?
|
|
+ if not cls.check_skill(skill_id):
|
|
+ # we missed adding this skill in our tracking
|
|
+ cls.add_active_skill(skill_id)
|
|
+ for skill in cls.active_skills:
|
|
+ if skill[0] not in skill_ids:
|
|
+ # we missed removing this skill in our tracking
|
|
+ cls.remove_active_skill(skill[0])
|
|
+
|
|
+ # https://github.com/MycroftAI/mycroft-core/pull/1468
|
|
+ @classmethod
|
|
+ def remove_active_skill(cls, skill_id, silent=False):
|
|
+ """
|
|
+ Emits "converse.skill.deactivated" event, improvement of #1468
|
|
+ """
|
|
+ for skill in list(cls.active_skills):
|
|
+ if skill[0] == skill_id:
|
|
+ cls.active_skills.remove(skill)
|
|
+ if not silent:
|
|
+ cls.bus.emit(Message("converse.skill.deactivated",
|
|
+ {"skill_id": skill[0]}))
|
|
+
|
|
+ @classmethod
|
|
+ def add_active_skill(cls, skill_id):
|
|
+ """
|
|
+ Emits "converse.skill.activated" event, improvement of #1468
|
|
+ """
|
|
+ # search the list for an existing entry that already contains it
|
|
+ # and remove that reference
|
|
+ if skill_id != '':
|
|
+ cls.remove_active_skill(skill_id, silent=True)
|
|
+ # add skill with timestamp to start of skill_list
|
|
+ cls.active_skills.insert(0, [skill_id, time.time()])
|
|
+ # this might be sent more than once and it's perfectly fine
|
|
+ # it's just a new info message not consumed anywhere by default
|
|
+ cls.bus.emit(Message("converse.skill.activated",
|
|
+ {"skill_id": skill_id}))
|
|
+ else:
|
|
+ LOG.warning('Skill ID was empty, won\'t add to list of '
|
|
+ 'active skills.')
|
|
+
|
|
+ # status tracking
|
|
+ @classmethod
|
|
+ def handle_activate_request(cls, message):
|
|
+ """
|
|
+ a skill bumped itself to the top of active skills list
|
|
+ duplicate functionality from mycroft-core, keeping list in sync
|
|
+ """
|
|
+ skill_id = message.data["skill_id"]
|
|
+ cls.add_active_skill(skill_id)
|
|
+
|
|
+ @classmethod
|
|
+ def handle_converse_error(cls, message):
|
|
+ """
|
|
+ a skill was removed from active skill list due to converse error
|
|
+ duplicate functionality from mycroft-core, keeping list in sync
|
|
+ """
|
|
+ skill_id = message.data["skill_id"]
|
|
+ if message.data["error"] == "skill id does not exist":
|
|
+ cls.remove_active_skill(skill_id)
|
|
+
|
|
+ @classmethod
|
|
+ def handle_intent_start(cls, message):
|
|
+ """
|
|
+ duplicate functionality from mycroft-core, keeping list in sync
|
|
+
|
|
+ TODO skill_id from message, core is not passing it along... it used
|
|
+ to be possible to retrieve it from munged message but that changed.
|
|
+ send a PR (if those got merged this code wouldn't exist)
|
|
+
|
|
+ handle_utterance will take over this functionality for now
|
|
+ handle_converse_response will take corrective action
|
|
+ """
|
|
+ # skill_id = message.data["skill_id"]
|
|
+ # bump skill to top of active list
|
|
+ # cls.add_active_skill(skill_id)
|
|
+
|
|
+ @classmethod
|
|
+ def handle_utterance(cls, message):
|
|
+ """
|
|
+ duplicate functionality from mycroft-core, keeping list in sync
|
|
+
|
|
+ WORKAROUND - skill_id missing in handle_intent_start, will keep list
|
|
+ in sync by using the IntentAPI to check what skill the utterance
|
|
+ should trigger
|
|
+
|
|
+ handle_converse_response will take corrective action
|
|
+ """
|
|
+ # NOTE borked in mycroft-core
|
|
+ # needs https://github.com/MycroftAI/mycroft-core/pull/2786
|
|
+ skill_id = cls.intent_api.get_skill(message.data["utterances"][0])
|
|
+ if skill_id:
|
|
+ # this skill will trigger and therefore is the last active skill
|
|
+ cls.add_active_skill(skill_id)
|
|
+ # will remove expired intents from list
|
|
+ cls.filter_active_skills()
|
|
+
|
|
+ @classmethod
|
|
+ def handle_converse_response(cls, message):
|
|
+ """
|
|
+ tracks last_conversed skill
|
|
+
|
|
+ FAILSAFE - additional checks to correct active skills list,
|
|
+ but that should never happen, accounts for mistakes in
|
|
+ handle_utterance / intent_api
|
|
+
|
|
+ accounts for https://github.com/MycroftAI/mycroft-core/pull/2786
|
|
+ not yet being merged
|
|
+ """
|
|
+ skill_id = message.data["skill_id"]
|
|
+ if 'error' in message.data:
|
|
+ cls.handle_converse_error(message)
|
|
+ elif message.data.get('result') is True:
|
|
+ cls.last_conversed = skill_id
|
|
+ if not cls.check_skill(skill_id):
|
|
+ # seems like we missed adding this skill to active list
|
|
+ # NOTE this is a failsafe and should never trigger
|
|
+ # since this answered True we have the real timestamp
|
|
+ cls.add_active_skill(skill_id)
|
|
+ elif not cls.check_skill(skill_id):
|
|
+ # seems like we missed adding this skill to active list
|
|
+ # NOTE this is a failsafe and should never trigger
|
|
+ # since this answered false and we don't have the real timestamp
|
|
+ # let's add it to the end of the active_skills list
|
|
+ ts = time.time()
|
|
+ if len(cls.active_skills):
|
|
+ ts = cls.active_skills[-1][1]
|
|
+ cls.active_skills.append([skill_id, ts])
|
|
diff --git a/ovos_utils/intents/intent_service_interface.py b/ovos_utils/intents/intent_service_interface.py
|
|
new file mode 100644
|
|
index 0000000..25eaceb
|
|
--- /dev/null
|
|
+++ b/ovos_utils/intents/intent_service_interface.py
|
|
@@ -0,0 +1,495 @@
|
|
+from os.path import exists, isfile
|
|
+
|
|
+from adapt.intent import Intent
|
|
+from mycroft_bus_client import MessageBusClient
|
|
+from mycroft_bus_client.message import Message, dig_for_message
|
|
+from ovos_utils import create_daemon
|
|
+from ovos_utils.log import LOG
|
|
+
|
|
+
|
|
+def to_alnum(skill_id):
|
|
+ """Convert a skill id to only alphanumeric characters
|
|
+
|
|
+ Non alpha-numeric characters are converted to "_"
|
|
+
|
|
+ Args:
|
|
+ skill_id (str): identifier to be converted
|
|
+ Returns:
|
|
+ (str) String of letters
|
|
+ """
|
|
+ return ''.join(c if c.isalnum() else '_' for c in str(skill_id))
|
|
+
|
|
+
|
|
+def munge_regex(regex, skill_id):
|
|
+ """Insert skill id as letters into match groups.
|
|
+
|
|
+ Args:
|
|
+ regex (str): regex string
|
|
+ skill_id (str): skill identifier
|
|
+ Returns:
|
|
+ (str) munged regex
|
|
+ """
|
|
+ base = '(?P<' + to_alnum(skill_id)
|
|
+ return base.join(regex.split('(?P<'))
|
|
+
|
|
+
|
|
+def munge_intent_parser(intent_parser, name, skill_id):
|
|
+ """Rename intent keywords to make them skill exclusive
|
|
+ This gives the intent parser an exclusive name in the
|
|
+ format <skill_id>:<name>. The keywords are given unique
|
|
+ names in the format <Skill id as letters><Intent name>.
|
|
+
|
|
+ The function will not munge instances that's already been
|
|
+ munged
|
|
+
|
|
+ Args:
|
|
+ intent_parser: (IntentParser) object to update
|
|
+ name: (str) Skill name
|
|
+ skill_id: (int) skill identifier
|
|
+ """
|
|
+ # Munge parser name
|
|
+ if not name.startswith(str(skill_id) + ':'):
|
|
+ intent_parser.name = str(skill_id) + ':' + name
|
|
+ else:
|
|
+ intent_parser.name = name
|
|
+
|
|
+ # Munge keywords
|
|
+ skill_id = to_alnum(skill_id)
|
|
+ # Munge required keyword
|
|
+ reqs = []
|
|
+ for i in intent_parser.requires:
|
|
+ if not i[0].startswith(skill_id):
|
|
+ kw = (skill_id + i[0], skill_id + i[0])
|
|
+ reqs.append(kw)
|
|
+ else:
|
|
+ reqs.append(i)
|
|
+ intent_parser.requires = reqs
|
|
+
|
|
+ # Munge optional keywords
|
|
+ opts = []
|
|
+ for i in intent_parser.optional:
|
|
+ if not i[0].startswith(skill_id):
|
|
+ kw = (skill_id + i[0], skill_id + i[0])
|
|
+ opts.append(kw)
|
|
+ else:
|
|
+ opts.append(i)
|
|
+ intent_parser.optional = opts
|
|
+
|
|
+ # Munge at_least_one keywords
|
|
+ at_least_one = []
|
|
+ for i in intent_parser.at_least_one:
|
|
+ element = [skill_id + e.replace(skill_id, '') for e in i]
|
|
+ at_least_one.append(tuple(element))
|
|
+ intent_parser.at_least_one = at_least_one
|
|
+
|
|
+
|
|
+class IntentServiceInterface:
|
|
+ """Interface to communicate with the Mycroft intent service.
|
|
+
|
|
+ This class wraps the messagebus interface of the intent service allowing
|
|
+ for easier interaction with the service. It wraps both the Adapt and
|
|
+ Padatious parts of the intent services.
|
|
+ """
|
|
+
|
|
+ def __init__(self, bus=None):
|
|
+ self.bus = bus
|
|
+ self.skill_id = self.__class__.__name__
|
|
+ self.registered_intents = []
|
|
+ self.detached_intents = []
|
|
+
|
|
+ def set_bus(self, bus):
|
|
+ self.bus = bus
|
|
+
|
|
+ def set_id(self, skill_id):
|
|
+ self.skill_id = skill_id
|
|
+
|
|
+ def register_adapt_keyword(self, vocab_type, entity, aliases=None,
|
|
+ lang=None):
|
|
+ """Send a message to the intent service to add an Adapt keyword.
|
|
+
|
|
+ vocab_type(str): Keyword reference
|
|
+ entity (str): Primary keyword
|
|
+ aliases (list): List of alternative kewords
|
|
+ """
|
|
+ aliases = aliases or []
|
|
+ msg = dig_for_message() or Message("")
|
|
+ if "skill_id" not in msg.context:
|
|
+ msg.context["skill_id"] = self.skill_id
|
|
+ self.bus.emit(msg.forward("register_vocab",
|
|
+ {'start': entity,
|
|
+ 'end': vocab_type,
|
|
+ 'lang': lang}))
|
|
+ for alias in aliases:
|
|
+ self.bus.emit(msg.forward("register_vocab",
|
|
+ {'start': alias,
|
|
+ 'end': vocab_type,
|
|
+ 'alias_of': entity,
|
|
+ 'lang': lang}))
|
|
+
|
|
+ def register_adapt_regex(self, regex, lang=None):
|
|
+ """Register a regex with the intent service.
|
|
+
|
|
+ Args:
|
|
+ regex (str): Regex to be registered, (Adapt extracts keyword
|
|
+ reference from named match group.
|
|
+ """
|
|
+ msg = dig_for_message() or Message("")
|
|
+ if "skill_id" not in msg.context:
|
|
+ msg.context["skill_id"] = self.skill_id
|
|
+ self.bus.emit(msg.forward("register_vocab",
|
|
+ {'regex': regex, 'lang': lang}))
|
|
+
|
|
+ def register_adapt_intent(self, name, intent_parser):
|
|
+ """Register an Adapt intent parser object.
|
|
+
|
|
+ Serializes the intent_parser and sends it over the messagebus to
|
|
+ registered.
|
|
+ """
|
|
+ msg = dig_for_message() or Message("")
|
|
+ if "skill_id" not in msg.context:
|
|
+ msg.context["skill_id"] = self.skill_id
|
|
+ self.bus.emit(msg.forward("register_intent", intent_parser.__dict__))
|
|
+ self.registered_intents.append((name, intent_parser))
|
|
+ self.detached_intents = [detached for detached in self.detached_intents
|
|
+ if detached[0] != name]
|
|
+
|
|
+ def detach_intent(self, intent_name):
|
|
+ """Remove an intent from the intent service.
|
|
+
|
|
+ The intent is saved in the list of detached intents for use when
|
|
+ re-enabling an intent.
|
|
+
|
|
+ Args:
|
|
+ intent_name(str): Intent reference
|
|
+ """
|
|
+ name = intent_name.split(':')[1]
|
|
+ if name in self:
|
|
+ msg = dig_for_message() or Message("")
|
|
+ if "skill_id" not in msg.context:
|
|
+ msg.context["skill_id"] = self.skill_id
|
|
+ self.bus.emit(msg.forward("detach_intent",
|
|
+ {"intent_name": intent_name}))
|
|
+ self.detached_intents.append((name, self.get_intent(name)))
|
|
+ self.registered_intents = [pair for pair in self.registered_intents
|
|
+ if pair[0] != name]
|
|
+
|
|
+ def set_adapt_context(self, context, word, origin):
|
|
+ """Set an Adapt context.
|
|
+
|
|
+ Args:
|
|
+ context (str): context keyword name
|
|
+ word (str): word to register
|
|
+ origin (str): original origin of the context (for cross context)
|
|
+ """
|
|
+ msg = dig_for_message() or Message("")
|
|
+ if "skill_id" not in msg.context:
|
|
+ msg.context["skill_id"] = self.skill_id
|
|
+ self.bus.emit(msg.forward('add_context',
|
|
+ {'context': context, 'word': word,
|
|
+ 'origin': origin}))
|
|
+
|
|
+ def remove_adapt_context(self, context):
|
|
+ """Remove an active Adapt context.
|
|
+
|
|
+ Args:
|
|
+ context(str): name of context to remove
|
|
+ """
|
|
+ msg = dig_for_message() or Message("")
|
|
+ if "skill_id" not in msg.context:
|
|
+ msg.context["skill_id"] = self.skill_id
|
|
+ self.bus.emit(msg.forward('remove_context', {'context': context}))
|
|
+
|
|
+ def register_padatious_intent(self, intent_name, filename, lang):
|
|
+ """Register a padatious intent file with Padatious.
|
|
+
|
|
+ Args:
|
|
+ intent_name(str): intent identifier
|
|
+ filename(str): complete file path for entity file
|
|
+ """
|
|
+ if not isinstance(filename, str):
|
|
+ raise ValueError('Filename path must be a string')
|
|
+ if not exists(filename):
|
|
+ raise FileNotFoundError('Unable to find "{}"'.format(filename))
|
|
+
|
|
+ data = {'file_name': filename,
|
|
+ 'name': intent_name,
|
|
+ 'lang': lang}
|
|
+ msg = dig_for_message() or Message("")
|
|
+ if "skill_id" not in msg.context:
|
|
+ msg.context["skill_id"] = self.skill_id
|
|
+ self.bus.emit(msg.forward("padatious:register_intent", data))
|
|
+ self.registered_intents.append((intent_name.split(':')[-1], data))
|
|
+
|
|
+ def register_padatious_entity(self, entity_name, filename, lang):
|
|
+ """Register a padatious entity file with Padatious.
|
|
+
|
|
+ Args:
|
|
+ entity_name(str): entity name
|
|
+ filename(str): complete file path for entity file
|
|
+ """
|
|
+ if not isinstance(filename, str):
|
|
+ raise ValueError('Filename path must be a string')
|
|
+ if not exists(filename):
|
|
+ raise FileNotFoundError('Unable to find "{}"'.format(filename))
|
|
+ msg = dig_for_message() or Message("")
|
|
+ if "skill_id" not in msg.context:
|
|
+ msg.context["skill_id"] = self.skill_id
|
|
+ self.bus.emit(msg.forward('padatious:register_entity',
|
|
+ {'file_name': filename,
|
|
+ 'name': entity_name,
|
|
+ 'lang': lang}))
|
|
+
|
|
+ def __iter__(self):
|
|
+ """Iterator over the registered intents.
|
|
+
|
|
+ Returns an iterator returning name-handler pairs of the registered
|
|
+ intent handlers.
|
|
+ """
|
|
+ return iter(self.registered_intents)
|
|
+
|
|
+ def __contains__(self, val):
|
|
+ """Checks if an intent name has been registered."""
|
|
+ return val in [i[0] for i in self.registered_intents]
|
|
+
|
|
+ def get_intent(self, intent_name):
|
|
+ """Get intent from intent_name.
|
|
+
|
|
+ This will find both enabled and disabled intents.
|
|
+
|
|
+ Args:
|
|
+ intent_name (str): name to find.
|
|
+
|
|
+ Returns:
|
|
+ Found intent or None if none were found.
|
|
+ """
|
|
+ for name, intent in self:
|
|
+ if name == intent_name:
|
|
+ return intent
|
|
+ for name, intent in self.detached_intents:
|
|
+ if name == intent_name:
|
|
+ return intent
|
|
+ return None
|
|
+
|
|
+
|
|
+class IntentQueryApi:
|
|
+ """
|
|
+ Query Intent Service at runtime
|
|
+ """
|
|
+
|
|
+ def __init__(self, bus=None, timeout=5):
|
|
+ if bus is None:
|
|
+ bus = MessageBusClient()
|
|
+ create_daemon(bus.run_forever)
|
|
+ self.bus = bus
|
|
+ self.timeout = timeout
|
|
+
|
|
+ def get_adapt_intent(self, utterance, lang="en-us"):
|
|
+ """ get best adapt intent for utterance """
|
|
+ msg = Message("intent.service.adapt.get",
|
|
+ {"utterance": utterance, "lang": lang},
|
|
+ context={"destination": "intent_service",
|
|
+ "source": "intent_api"})
|
|
+
|
|
+ resp = self.bus.wait_for_response(msg,
|
|
+ 'intent.service.adapt.reply',
|
|
+ timeout=self.timeout)
|
|
+ data = resp.data if resp is not None else {}
|
|
+ if not data:
|
|
+ LOG.error("Intent Service timed out!")
|
|
+ return None
|
|
+ return data["intent"]
|
|
+
|
|
+ def get_padatious_intent(self, utterance, lang="en-us"):
|
|
+ """ get best padatious intent for utterance """
|
|
+ msg = Message("intent.service.padatious.get",
|
|
+ {"utterance": utterance, "lang": lang},
|
|
+ context={"destination": "intent_service",
|
|
+ "source": "intent_api"})
|
|
+ resp = self.bus.wait_for_response(msg,
|
|
+ 'intent.service.padatious.reply',
|
|
+ timeout=self.timeout)
|
|
+ data = resp.data if resp is not None else {}
|
|
+ if not data:
|
|
+ LOG.error("Intent Service timed out!")
|
|
+ return None
|
|
+ return data["intent"]
|
|
+
|
|
+ def get_intent(self, utterance, lang="en-us"):
|
|
+ """ get best intent for utterance """
|
|
+ msg = Message("intent.service.intent.get",
|
|
+ {"utterance": utterance, "lang": lang},
|
|
+ context={"destination": "intent_service",
|
|
+ "source": "intent_api"})
|
|
+ resp = self.bus.wait_for_response(msg,
|
|
+ 'intent.service.intent.reply',
|
|
+ timeout=self.timeout)
|
|
+ data = resp.data if resp is not None else {}
|
|
+ if not data:
|
|
+ LOG.error("Intent Service timed out!")
|
|
+ return None
|
|
+ return data["intent"]
|
|
+
|
|
+ def get_skill(self, utterance, lang="en-us"):
|
|
+ """ get skill that utterance will trigger """
|
|
+ intent = self.get_intent(utterance, lang)
|
|
+ if not intent:
|
|
+ return None
|
|
+ # theoretically skill_id might be missing
|
|
+ if intent.get("skill_id"):
|
|
+ return intent["skill_id"]
|
|
+ # retrieve skill from munged intent name
|
|
+ if intent.get("intent_name"): # padatious + adapt
|
|
+ return intent["name"].split(":")[0]
|
|
+ if intent.get("intent_type"): # adapt
|
|
+ return intent["intent_type"].split(":")[0]
|
|
+ return None # raise some error here maybe? this should never happen
|
|
+
|
|
+ def get_skills_manifest(self):
|
|
+ msg = Message("intent.service.skills.get",
|
|
+ context={"destination": "intent_service",
|
|
+ "source": "intent_api"})
|
|
+ resp = self.bus.wait_for_response(msg,
|
|
+ 'intent.service.skills.reply',
|
|
+ timeout=self.timeout)
|
|
+ data = resp.data if resp is not None else {}
|
|
+ if not data:
|
|
+ LOG.error("Intent Service timed out!")
|
|
+ return None
|
|
+ return data["skills"]
|
|
+
|
|
+ def get_active_skills(self, include_timestamps=False):
|
|
+ msg = Message("intent.service.active_skills.get",
|
|
+ context={"destination": "intent_service",
|
|
+ "source": "intent_api"})
|
|
+ resp = self.bus.wait_for_response(msg,
|
|
+ 'intent.service.active_skills.reply',
|
|
+ timeout=self.timeout)
|
|
+ data = resp.data if resp is not None else {}
|
|
+ if not data:
|
|
+ LOG.error("Intent Service timed out!")
|
|
+ return None
|
|
+ if include_timestamps:
|
|
+ return data["skills"]
|
|
+ return [s[0] for s in data["skills"]]
|
|
+
|
|
+ def get_adapt_manifest(self):
|
|
+ msg = Message("intent.service.adapt.manifest.get",
|
|
+ context={"destination": "intent_service",
|
|
+ "source": "intent_api"})
|
|
+ resp = self.bus.wait_for_response(msg,
|
|
+ 'intent.service.adapt.manifest',
|
|
+ timeout=self.timeout)
|
|
+ data = resp.data if resp is not None else {}
|
|
+ if not data:
|
|
+ LOG.error("Intent Service timed out!")
|
|
+ return None
|
|
+ return data["intents"]
|
|
+
|
|
+ def get_padatious_manifest(self):
|
|
+ msg = Message("intent.service.padatious.manifest.get",
|
|
+ context={"destination": "intent_service",
|
|
+ "source": "intent_api"})
|
|
+ resp = self.bus.wait_for_response(msg,
|
|
+ 'intent.service.padatious.manifest',
|
|
+ timeout=self.timeout)
|
|
+ data = resp.data if resp is not None else {}
|
|
+ if not data:
|
|
+ LOG.error("Intent Service timed out!")
|
|
+ return None
|
|
+ return data["intents"]
|
|
+
|
|
+ def get_intent_manifest(self):
|
|
+ padatious = self.get_padatious_manifest()
|
|
+ adapt = self.get_adapt_manifest()
|
|
+ return {"adapt": adapt,
|
|
+ "padatious": padatious}
|
|
+
|
|
+ def get_vocab_manifest(self):
|
|
+ msg = Message("intent.service.adapt.vocab.manifest.get",
|
|
+ context={"destination": "intent_service",
|
|
+ "source": "intent_api"})
|
|
+ reply_msg_type = 'intent.service.adapt.vocab.manifest'
|
|
+ resp = self.bus.wait_for_response(msg,
|
|
+ reply_msg_type,
|
|
+ timeout=self.timeout)
|
|
+ data = resp.data if resp is not None else {}
|
|
+ if not data:
|
|
+ LOG.error("Intent Service timed out!")
|
|
+ return None
|
|
+
|
|
+ vocab = {}
|
|
+ for voc in data["vocab"]:
|
|
+ if voc.get("regex"):
|
|
+ continue
|
|
+ if voc["end"] not in vocab:
|
|
+ vocab[voc["end"]] = {"samples": []}
|
|
+ vocab[voc["end"]]["samples"].append(voc["start"])
|
|
+ return [{"name": voc, "samples": vocab[voc]["samples"]}
|
|
+ for voc in vocab]
|
|
+
|
|
+ def get_regex_manifest(self):
|
|
+ msg = Message("intent.service.adapt.vocab.manifest.get",
|
|
+ context={"destination": "intent_service",
|
|
+ "source": "intent_api"})
|
|
+ reply_msg_type = 'intent.service.adapt.vocab.manifest'
|
|
+ resp = self.bus.wait_for_response(msg,
|
|
+ reply_msg_type,
|
|
+ timeout=self.timeout)
|
|
+ data = resp.data if resp is not None else {}
|
|
+ if not data:
|
|
+ LOG.error("Intent Service timed out!")
|
|
+ return None
|
|
+
|
|
+ vocab = {}
|
|
+ for voc in data["vocab"]:
|
|
+ if not voc.get("regex"):
|
|
+ continue
|
|
+ name = voc["regex"].split("(?P<")[-1].split(">")[0]
|
|
+ if name not in vocab:
|
|
+ vocab[name] = {"samples": []}
|
|
+ vocab[name]["samples"].append(voc["regex"])
|
|
+ return [{"name": voc, "regexes": vocab[voc]["samples"]}
|
|
+ for voc in vocab]
|
|
+
|
|
+ def get_entities_manifest(self):
|
|
+ msg = Message("intent.service.padatious.entities.manifest.get",
|
|
+ context={"destination": "intent_service",
|
|
+ "source": "intent_api"})
|
|
+ reply_msg_type = 'intent.service.padatious.entities.manifest'
|
|
+ resp = self.bus.wait_for_response(msg,
|
|
+ reply_msg_type,
|
|
+ timeout=self.timeout)
|
|
+ data = resp.data if resp is not None else {}
|
|
+ if not data:
|
|
+ LOG.error("Intent Service timed out!")
|
|
+ return None
|
|
+
|
|
+ entities = []
|
|
+ # read files
|
|
+ for ent in data["entities"]:
|
|
+ if isfile(ent["file_name"]):
|
|
+ with open(ent["file_name"]) as f:
|
|
+ lines = f.read().replace("(", "").replace(")", "").split(
|
|
+ "\n")
|
|
+ samples = []
|
|
+ for l in lines:
|
|
+ samples += [a.strip() for a in l.split("|") if a.strip()]
|
|
+ entities.append({"name": ent["name"], "samples": samples})
|
|
+ return entities
|
|
+
|
|
+ def get_keywords_manifest(self):
|
|
+ padatious = self.get_entities_manifest()
|
|
+ adapt = self.get_vocab_manifest()
|
|
+ regex = self.get_regex_manifest()
|
|
+ return {"adapt": adapt,
|
|
+ "padatious": padatious,
|
|
+ "regex": regex}
|
|
+
|
|
+
|
|
+def open_intent_envelope(message):
|
|
+ """Convert dictionary received over messagebus to Intent."""
|
|
+ intent_dict = message.data
|
|
+ return Intent(intent_dict.get('name'),
|
|
+ intent_dict.get('requires'),
|
|
+ intent_dict.get('at_least_one'),
|
|
+ intent_dict.get('optional'))
|
|
|
|
From c3b7ce30d1060251d1f3db894adcc780f10a7a1f Mon Sep 17 00:00:00 2001
|
|
From: jarbasal <jarbasai@mailfence.com>
|
|
Date: Thu, 30 Sep 2021 00:55:13 +0100
|
|
Subject: [PATCH 2/5] dialog utils
|
|
|
|
---
|
|
ovos_utils/dialog.py | 193 ++++++++++++++++++++++++++++++++++++
|
|
ovos_utils/lang/__init__.py | 26 +++++
|
|
2 files changed, 219 insertions(+)
|
|
create mode 100644 ovos_utils/dialog.py
|
|
|
|
diff --git a/ovos_utils/dialog.py b/ovos_utils/dialog.py
|
|
new file mode 100644
|
|
index 0000000..7b51900
|
|
--- /dev/null
|
|
+++ b/ovos_utils/dialog.py
|
|
@@ -0,0 +1,193 @@
|
|
+import os
|
|
+import random
|
|
+import re
|
|
+from os.path import join
|
|
+from pathlib import Path
|
|
+
|
|
+from ovos_utils.bracket_expansion import expand_options
|
|
+from ovos_utils.configuration import read_mycroft_config
|
|
+from ovos_utils.file_utils import resolve_resource_file
|
|
+from ovos_utils.lang import translate_word
|
|
+from ovos_utils.log import LOG
|
|
+
|
|
+
|
|
+class MustacheDialogRenderer:
|
|
+ """A dialog template renderer based on the mustache templating language."""
|
|
+
|
|
+ def __init__(self):
|
|
+ self.templates = {}
|
|
+ self.recent_phrases = []
|
|
+
|
|
+ # TODO magic numbers are bad!
|
|
+ self.max_recent_phrases = 3
|
|
+ # We cycle through lines in .dialog files to keep Mycroft from
|
|
+ # repeating the same phrase over and over. However, if a .dialog
|
|
+ # file only contains a few entries, this can cause it to loop.
|
|
+ #
|
|
+ # This offset will override max_recent_phrases on very short .dialog
|
|
+ # files. With the offset at 2, .dialog files with 3 or more lines will
|
|
+ # be managed to avoid repetition, but .dialog files with only 1 or 2
|
|
+ # lines will be unaffected. Dialog should never get stuck in a loop.
|
|
+ self.loop_prevention_offset = 2
|
|
+
|
|
+ def load_template_file(self, template_name, filename):
|
|
+ """Load a template by file name into the templates cache.
|
|
+
|
|
+ Args:
|
|
+ template_name (str): a unique identifier for a group of templates
|
|
+ filename (str): a fully qualified filename of a mustache template.
|
|
+ """
|
|
+ with open(filename, 'r', encoding='utf8') as f:
|
|
+ for line in f:
|
|
+ template_text = line.strip()
|
|
+ # Skip all lines starting with '#' and all empty lines
|
|
+ if (not template_text.startswith('#') and
|
|
+ template_text != ''):
|
|
+ if template_name not in self.templates:
|
|
+ self.templates[template_name] = []
|
|
+
|
|
+ # convert to standard python format string syntax. From
|
|
+ # double (or more) '{' followed by any number of
|
|
+ # whitespace followed by actual key followed by any number
|
|
+ # of whitespace followed by double (or more) '}'
|
|
+ template_text = re.sub(r'\{\{+\s*(.*?)\s*\}\}+', r'{\1}',
|
|
+ template_text)
|
|
+
|
|
+ self.templates[template_name].append(template_text)
|
|
+
|
|
+ def render(self, template_name, context=None, index=None):
|
|
+ """
|
|
+ Given a template name, pick a template and render it using the context.
|
|
+ If no matching template exists use template_name as template.
|
|
+
|
|
+ Tries not to let Mycroft say exactly the same thing twice in a row.
|
|
+
|
|
+ Args:
|
|
+ template_name (str): the name of a template group.
|
|
+ context (dict): dictionary representing values to be rendered
|
|
+ index (int): optional, the specific index in the collection of
|
|
+ templates
|
|
+
|
|
+ Returns:
|
|
+ str: the rendered string
|
|
+ """
|
|
+ context = context or {}
|
|
+ if template_name not in self.templates:
|
|
+ # When not found, return the name itself as the dialog
|
|
+ # This allows things like render("record.not.found") to either
|
|
+ # find a translation file "record.not.found.dialog" or return
|
|
+ # "record not found" literal.
|
|
+ return template_name.replace('.', ' ')
|
|
+
|
|
+ # Get the .dialog file's contents, minus any which have been spoken
|
|
+ # recently.
|
|
+ template_functions = self.templates.get(template_name)
|
|
+
|
|
+ if index is None:
|
|
+ template_functions = ([t for t in template_functions
|
|
+ if t not in self.recent_phrases] or
|
|
+ template_functions)
|
|
+ line = random.choice(template_functions)
|
|
+ else:
|
|
+ line = template_functions[index % len(template_functions)]
|
|
+ # Replace {key} in line with matching values from context
|
|
+ line = line.format(**context)
|
|
+ line = random.choice(expand_options(line))
|
|
+
|
|
+ # Here's where we keep track of what we've said recently. Remember,
|
|
+ # this is by line in the .dialog file, not by exact phrase
|
|
+ self.recent_phrases.append(line)
|
|
+ if (len(self.recent_phrases) >
|
|
+ min(self.max_recent_phrases, len(self.templates.get(
|
|
+ template_name)) - self.loop_prevention_offset)):
|
|
+ self.recent_phrases.pop(0)
|
|
+ return line
|
|
+
|
|
+
|
|
+def load_dialogs(dialog_dir, renderer=None):
|
|
+ """Load all dialog files within the specified directory.
|
|
+
|
|
+ Args:
|
|
+ dialog_dir (str): directory that contains dialog files
|
|
+
|
|
+ Returns:
|
|
+ a loaded instance of a dialog renderer
|
|
+ """
|
|
+ if renderer is None:
|
|
+ renderer = MustacheDialogRenderer()
|
|
+
|
|
+ directory = Path(dialog_dir)
|
|
+ if not directory.exists() or not directory.is_dir():
|
|
+ LOG.warning('No dialog files found: {}'.format(dialog_dir))
|
|
+ return renderer
|
|
+
|
|
+ for path, _, files in os.walk(str(directory)):
|
|
+ for f in files:
|
|
+ if f.endswith('.dialog'):
|
|
+ renderer.load_template_file(f.replace('.dialog', ''),
|
|
+ join(path, f))
|
|
+ return renderer
|
|
+
|
|
+
|
|
+def get_dialog(phrase, lang=None, context=None):
|
|
+ """Looks up a resource file for the given phrase.
|
|
+
|
|
+ If no file is found, the requested phrase is returned as the string. This
|
|
+ will use the default language for translations.
|
|
+
|
|
+ Args:
|
|
+ phrase (str): resource phrase to retrieve/translate
|
|
+ lang (str): the language to use
|
|
+ context (dict): values to be inserted into the string
|
|
+
|
|
+ Returns:
|
|
+ str: a randomized and/or translated version of the phrase
|
|
+ """
|
|
+
|
|
+ if not lang:
|
|
+ try:
|
|
+ conf = read_mycroft_config()
|
|
+ lang = conf.get('lang')
|
|
+ except FileNotFoundError:
|
|
+ lang = "en-us"
|
|
+
|
|
+ filename = join('text', lang.lower(), phrase + '.dialog')
|
|
+ template = resolve_resource_file(filename)
|
|
+ if not template:
|
|
+ LOG.debug('Resource file not found: {}'.format(filename))
|
|
+ return phrase
|
|
+
|
|
+ stache = MustacheDialogRenderer()
|
|
+ stache.load_template_file('template', template)
|
|
+ if not context:
|
|
+ context = {}
|
|
+ return stache.render('template', context)
|
|
+
|
|
+
|
|
+def join_list(items, connector, sep=None, lang=''):
|
|
+ """ Join a list into a phrase using the given connector word
|
|
+ Examples:
|
|
+ join_list([1,2,3], "and") -> "1, 2 and 3"
|
|
+ join_list([1,2,3], "and", ";") -> "1; 2 and 3"
|
|
+ Args:
|
|
+ items (array): items to be joined
|
|
+ connector (str): connecting word (resource name), like "and" or "or"
|
|
+ sep (str, optional): separator character, default = ","
|
|
+ lang (str, optional): an optional BCP-47 language code, if omitted
|
|
+ the default language will be used.
|
|
+ Returns:
|
|
+ str: the connected list phrase
|
|
+ """
|
|
+
|
|
+ if not items:
|
|
+ return ""
|
|
+ if len(items) == 1:
|
|
+ return str(items[0])
|
|
+
|
|
+ if not sep:
|
|
+ sep = ", "
|
|
+ else:
|
|
+ sep += " "
|
|
+ return (sep.join(str(item) for item in items[:-1]) +
|
|
+ " " + translate_word(connector, lang) +
|
|
+ " " + items[-1])
|
|
diff --git a/ovos_utils/lang/__init__.py b/ovos_utils/lang/__init__.py
|
|
index 28c8235..b43ccae 100644
|
|
--- a/ovos_utils/lang/__init__.py
|
|
+++ b/ovos_utils/lang/__init__.py
|
|
@@ -2,6 +2,7 @@
|
|
from os import system, makedirs, listdir
|
|
from ovos_utils.lang.detect import detect_lang
|
|
from ovos_utils.lang.translate import translate_text
|
|
+from ovos_utils.file_utils import resolve_resource_file
|
|
|
|
|
|
def get_tts(sentence, lang="en-us", mp3_file="/tmp/google_tx_tts.mp3"):
|
|
@@ -42,3 +43,28 @@ def get_language_dir(base_path, lang="en-us"):
|
|
if len(paths):
|
|
return paths[0]
|
|
return join(base_path, lang)
|
|
+
|
|
+
|
|
+def translate_word(name, lang='en-us'):
|
|
+ """ Helper to get word translations
|
|
+ Args:
|
|
+ name (str): Word name. Returned as the default value if not translated
|
|
+ lang (str, optional): an optional BCP-47 language code, if omitted
|
|
+ the default language will be used.
|
|
+ Returns:
|
|
+ str: translated version of resource name
|
|
+ """
|
|
+ filename = resolve_resource_file(join("text", lang, name + ".word"))
|
|
+ if filename:
|
|
+ # open the file
|
|
+ try:
|
|
+ with open(filename, 'r', encoding='utf8') as f:
|
|
+ for line in f:
|
|
+ word = line.strip()
|
|
+ if word.startswith("#"):
|
|
+ continue # skip comment lines
|
|
+ return word
|
|
+ except Exception:
|
|
+ pass
|
|
+ return name # use resource name as the word
|
|
+
|
|
|
|
From 3b271d6fa3c95b311b11438ab63be7ac1371e6ed Mon Sep 17 00:00:00 2001
|
|
From: jarbasal <jarbasai@mailfence.com>
|
|
Date: Thu, 30 Sep 2021 06:17:20 +0100
|
|
Subject: [PATCH 3/5] circular import
|
|
|
|
---
|
|
ovos_utils/file_utils.py | 5 ++++-
|
|
1 file changed, 4 insertions(+), 1 deletion(-)
|
|
|
|
diff --git a/ovos_utils/file_utils.py b/ovos_utils/file_utils.py
|
|
index de9122a..9b3c44b 100644
|
|
--- a/ovos_utils/file_utils.py
|
|
+++ b/ovos_utils/file_utils.py
|
|
@@ -6,7 +6,6 @@
|
|
from os.path import splitext, join, dirname
|
|
|
|
from ovos_utils.bracket_expansion import expand_options
|
|
-from ovos_utils.intents.intent_service_interface import to_alnum, munge_regex
|
|
from ovos_utils.log import LOG
|
|
|
|
|
|
@@ -135,6 +134,8 @@ def load_regex_from_file(path, skill_id):
|
|
path: path to vocabulary file (*.voc)
|
|
skill_id: skill_id to the regex is tied to
|
|
"""
|
|
+ from ovos_utils.intents.intent_service_interface import munge_regex
|
|
+
|
|
regexes = []
|
|
if path.endswith('.rx'):
|
|
with open(path, 'r', encoding='utf8') as reg_file:
|
|
@@ -163,6 +164,8 @@ def load_vocabulary(basedir, skill_id):
|
|
Returns:
|
|
dict with intent_type as keys and list of list of lists as value.
|
|
"""
|
|
+ from ovos_utils.intents.intent_service_interface import to_alnum
|
|
+
|
|
vocabs = {}
|
|
for path, _, files in walk(basedir):
|
|
for f in files:
|
|
|
|
From 1771ed406432ae85cae172c19e255f915c730a7d Mon Sep 17 00:00:00 2001
|
|
From: jarbasal <jarbasai@mailfence.com>
|
|
Date: Thu, 30 Sep 2021 16:43:59 +0100
|
|
Subject: [PATCH 4/5] delayed bus init
|
|
|
|
---
|
|
ovos_utils/gui.py | 32 ++++++++++++++++++++++++++++----
|
|
1 file changed, 28 insertions(+), 4 deletions(-)
|
|
|
|
diff --git a/ovos_utils/gui.py b/ovos_utils/gui.py
|
|
index 9b4a472..ea62f9c 100644
|
|
--- a/ovos_utils/gui.py
|
|
+++ b/ovos_utils/gui.py
|
|
@@ -429,7 +429,7 @@ class GUIInterface:
|
|
"""
|
|
|
|
def __init__(self, skill_id, bus=None, remote_server=None, config=None):
|
|
- self.bus = bus or get_mycroft_bus()
|
|
+ self.bus = bus
|
|
self.__session_data = {} # synced to GUI for use by this skill's pages
|
|
self.pages = []
|
|
self.current_page_idx = -1
|
|
@@ -437,6 +437,11 @@ def __init__(self, skill_id, bus=None, remote_server=None, config=None):
|
|
self.on_gui_changed_callback = None
|
|
self.remote_url = remote_server
|
|
self._events = []
|
|
+ if bus:
|
|
+ self.set_bus(bus)
|
|
+
|
|
+ def set_bus(self, bus=None):
|
|
+ self.bus = bus or get_mycroft_bus()
|
|
self.setup_default_handlers()
|
|
|
|
@property
|
|
@@ -448,6 +453,8 @@ def page(self):
|
|
def connected(self):
|
|
"""Returns True if at least 1 remote gui is connected or if gui is
|
|
installed and running locally, else False"""
|
|
+ if not self.bus:
|
|
+ return False
|
|
return can_use_gui(self.bus)
|
|
|
|
def build_message_type(self, event):
|
|
@@ -475,6 +482,8 @@ def register_handler(self, event, handler):
|
|
event (str): event to catch
|
|
handler: function to handle the event
|
|
"""
|
|
+ if not self.bus:
|
|
+ raise RuntimeError("bus not set, did you call self.bind() ?")
|
|
event = self.build_message_type(event)
|
|
self._events.append((event, handler))
|
|
self.bus.on(event, handler)
|
|
@@ -501,6 +510,8 @@ def gui_set(self, message):
|
|
self.on_gui_changed_callback()
|
|
|
|
def _sync_data(self):
|
|
+ if not self.bus:
|
|
+ raise RuntimeError("bus not set, did you call self.bind() ?")
|
|
data = self.__session_data.copy()
|
|
data.update({'__from': self.skill_id})
|
|
self.bus.emit(Message("gui.value.set", data))
|
|
@@ -539,6 +550,8 @@ def clear(self):
|
|
self.__session_data = {}
|
|
self.pages = []
|
|
self.current_page_idx = -1
|
|
+ if not self.bus:
|
|
+ raise RuntimeError("bus not set, did you call self.bind() ?")
|
|
self.bus.emit(Message("gui.clear.namespace",
|
|
{"__from": self.skill_id}))
|
|
|
|
@@ -551,6 +564,8 @@ def send_event(self, event_name, params=None):
|
|
should be sent along with the request.
|
|
"""
|
|
params = params or {}
|
|
+ if not self.bus:
|
|
+ raise RuntimeError("bus not set, did you call self.bind() ?")
|
|
self.bus.emit(Message("gui.event.send",
|
|
{"__from": self.skill_id,
|
|
"event_name": event_name,
|
|
@@ -610,6 +625,8 @@ def show_pages(self, page_names, index=0, override_idle=None,
|
|
True: Disables showing all platform skill animations.
|
|
False: 'Default' always show animations.
|
|
"""
|
|
+ if not self.bus:
|
|
+ raise RuntimeError("bus not set, did you call self.bind() ?")
|
|
if isinstance(page_names, str):
|
|
page_names = [page_names]
|
|
if not isinstance(page_names, list):
|
|
@@ -649,6 +666,8 @@ def remove_pages(self, page_names):
|
|
page_names (list): List of page names (str) to display, such as
|
|
["Weather.qml", "Forecast.qml", "Other.qml"]
|
|
"""
|
|
+ if not self.bus:
|
|
+ raise RuntimeError("bus not set, did you call self.bind() ?")
|
|
if not isinstance(page_names, list):
|
|
page_names = [page_names]
|
|
page_urls = self._pages2uri(page_names)
|
|
@@ -671,6 +690,8 @@ def show_notification(self, content, action=None,
|
|
transient: 'Default' displays a notification with a timeout.
|
|
sticky: displays a notification that sticks to the screen.
|
|
"""
|
|
+ if not self.bus:
|
|
+ raise RuntimeError("bus not set, did you call self.bind() ?")
|
|
self.bus.emit(Message("homescreen.notification.set",
|
|
data={
|
|
"sender": self.skill_id,
|
|
@@ -794,6 +815,8 @@ def release(self):
|
|
allow different platforms to properly handle this event.
|
|
Also calls self.clear() to reset the state variables
|
|
Platforms can close the window or go back to previous page"""
|
|
+ if not self.bus:
|
|
+ raise RuntimeError("bus not set, did you call self.bind() ?")
|
|
self.clear()
|
|
self.bus.emit(Message("mycroft.gui.screen.close",
|
|
{"skill_id": self.skill_id}))
|
|
@@ -803,9 +826,10 @@ def shutdown(self):
|
|
|
|
Clear pages loaded through this interface and remove the bus events
|
|
"""
|
|
- self.release()
|
|
- for event, handler in self._events:
|
|
- self.bus.remove(event, handler)
|
|
+ if self.bus:
|
|
+ self.release()
|
|
+ for event, handler in self._events:
|
|
+ self.bus.remove(event, handler)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
From 6db561b684ee88fbf3bb9bd9906bf25d95432873 Mon Sep 17 00:00:00 2001
|
|
From: jarbasal <jarbasai@mailfence.com>
|
|
Date: Fri, 1 Oct 2021 15:46:26 +0100
|
|
Subject: [PATCH 5/5] added some helper functions for intent handling /
|
|
disabling
|
|
|
|
---
|
|
ovos_utils/intents/intent_service_interface.py | 16 +++++++++++++++-
|
|
1 file changed, 15 insertions(+), 1 deletion(-)
|
|
|
|
diff --git a/ovos_utils/intents/intent_service_interface.py b/ovos_utils/intents/intent_service_interface.py
|
|
index 25eaceb..a014880 100644
|
|
--- a/ovos_utils/intents/intent_service_interface.py
|
|
+++ b/ovos_utils/intents/intent_service_interface.py
|
|
@@ -162,7 +162,12 @@ def detach_intent(self, intent_name):
|
|
Args:
|
|
intent_name(str): Intent reference
|
|
"""
|
|
- name = intent_name.split(':')[1]
|
|
+ # split skill_id if needed
|
|
+ if intent_name not in self and ":" in intent_name:
|
|
+ name = intent_name.split(':')[1]
|
|
+ else:
|
|
+ name = intent_name
|
|
+
|
|
if name in self:
|
|
msg = dig_for_message() or Message("")
|
|
if "skill_id" not in msg.context:
|
|
@@ -251,6 +256,15 @@ def __contains__(self, val):
|
|
"""Checks if an intent name has been registered."""
|
|
return val in [i[0] for i in self.registered_intents]
|
|
|
|
+ def get_intent_names(self):
|
|
+ return [i[0] for i in self.registered_intents]
|
|
+
|
|
+ def detach_all(self):
|
|
+ for name in self.get_intent_names():
|
|
+ self.detach_intent(name)
|
|
+ self.registered_intents = []
|
|
+ self.detached_intents = []
|
|
+
|
|
def get_intent(self, intent_name):
|
|
"""Get intent from intent_name.
|
|
|