Remove Python3.9 specific mycroft patches
This commit is contained in:
j1nx
parent
417c07ba75
commit
0209d97216
|
|
@ -1,41 +0,0 @@
|
||||||
From 002b16ba1f217a0b57e2d89ef51bda0fd94ab4f4 Mon Sep 17 00:00:00 2001
|
|
||||||
From: =?UTF-8?q?=C3=85ke=20Forslund?= <ake.forslund@gmail.com>
|
|
||||||
Date: Sat, 6 Mar 2021 11:11:14 +0100
|
|
||||||
Subject: [PATCH] Replace multiprocessing with concurrent.futures
|
|
||||||
|
|
||||||
This uses the ThreadPoolExecutor from concurrent.futures instead of
|
|
||||||
multiprocessings threadpool since the threadpool in multiprocessing
|
|
||||||
can't safely be used in a multithreaded context in Python 3.9+
|
|
||||||
---
|
|
||||||
msm/mycroft_skills_manager.py | 6 +++---
|
|
||||||
1 file changed, 3 insertions(+), 3 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/msm/mycroft_skills_manager.py b/msm/mycroft_skills_manager.py
|
|
||||||
index 82500b4..a1ae41f 100644
|
|
||||||
--- a/msm/mycroft_skills_manager.py
|
|
||||||
+++ b/msm/mycroft_skills_manager.py
|
|
||||||
@@ -23,12 +23,12 @@
|
|
||||||
|
|
||||||
MSM can be used on the command line but is also used by Mycroft core daemons.
|
|
||||||
"""
|
|
||||||
+from concurrent.futures import ThreadPoolExecutor
|
|
||||||
import time
|
|
||||||
import logging
|
|
||||||
import shutil
|
|
||||||
from functools import wraps
|
|
||||||
from glob import glob
|
|
||||||
-from multiprocessing.pool import ThreadPool
|
|
||||||
from os import path
|
|
||||||
from typing import Dict, List
|
|
||||||
|
|
||||||
@@ -487,8 +487,8 @@ def run_item(skill):
|
|
||||||
func.__name__, skill.name
|
|
||||||
))
|
|
||||||
|
|
||||||
- with ThreadPool(max_threads) as tp:
|
|
||||||
- return tp.map(run_item, skills)
|
|
||||||
+ with ThreadPoolExecutor(max_threads) as executor:
|
|
||||||
+ return executor.map(run_item, skills)
|
|
||||||
|
|
||||||
@save_device_skill_state
|
|
||||||
def install_defaults(self):
|
|
||||||
|
|
@ -1,38 +0,0 @@
|
||||||
From 9761899ea4d897b2c878d9a7ac300a611a0e4609 Mon Sep 17 00:00:00 2001
|
|
||||||
From: j1nx <p.steenbergen@j1nx.nl>
|
|
||||||
Date: Mon, 5 Apr 2021 17:05:47 +0200
|
|
||||||
Subject: [PATCH 1/1] Workaround of Python3.9+ multiprocessing issues
|
|
||||||
|
|
||||||
---
|
|
||||||
mycroft/util/process_utils.py | 15 +++++++++++++++
|
|
||||||
1 file changed, 15 insertions(+)
|
|
||||||
|
|
||||||
diff --git a/mycroft/util/process_utils.py b/mycroft/util/process_utils.py
|
|
||||||
index 81f5c90f59..d272b42729 100644
|
|
||||||
--- a/mycroft/util/process_utils.py
|
|
||||||
+++ b/mycroft/util/process_utils.py
|
|
||||||
@@ -10,6 +10,21 @@ from time import sleep
|
|
||||||
from .log import LOG
|
|
||||||
|
|
||||||
|
|
||||||
+def initialize_mp_context():
|
|
||||||
+ """Handle import behaviour changes in python 3.9+
|
|
||||||
+ Pre-import colliding multiprocessing parts.
|
|
||||||
+ This should be considered a workaround and not a solution of the issue.
|
|
||||||
+ """
|
|
||||||
+ LOG.info('Preloading multiprocessing internals...')
|
|
||||||
+ import multiprocessing.popen_spawn_posix # noqa
|
|
||||||
+ import multiprocessing.queues # noqa
|
|
||||||
+ import multiprocessing.resource_tracker # noqa
|
|
||||||
+ import multiprocessing.synchronize # noqa
|
|
||||||
+
|
|
||||||
+
|
|
||||||
+initialize_mp_context()
|
|
||||||
+
|
|
||||||
+
|
|
||||||
def reset_sigint_handler():
|
|
||||||
"""Reset the sigint handler to the default.
|
|
||||||
|
|
||||||
--
|
|
||||||
2.20.1
|
|
||||||
|
|
||||||
|
|
@ -1,99 +0,0 @@
|
||||||
From 4841ba2166107b14f2206ea4ae0cf7c68c62be1b Mon Sep 17 00:00:00 2001
|
|
||||||
From: =?UTF-8?q?=C3=85ke=20Forslund?= <ake.forslund@gmail.com>
|
|
||||||
Date: Sat, 6 Mar 2021 11:30:05 +0100
|
|
||||||
Subject: [PATCH 1/2] Replace multiprocessing with concurrent.futures
|
|
||||||
|
|
||||||
Multiprocessing pools can't safely be used in a threaded context in
|
|
||||||
Python 3.9+. This replaces the multiprocessing Pool with a
|
|
||||||
ProcessPoolExecutor from concurrent.futures.
|
|
||||||
|
|
||||||
Handle shutdown implicitly
|
|
||||||
---
|
|
||||||
padatious/training_manager.py | 15 +++++++++------
|
|
||||||
1 file changed, 9 insertions(+), 6 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/padatious/training_manager.py b/padatious/training_manager.py
|
|
||||||
index 3bef33c..61f2edd 100644
|
|
||||||
--- a/padatious/training_manager.py
|
|
||||||
+++ b/padatious/training_manager.py
|
|
||||||
@@ -11,9 +11,9 @@
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
-import multiprocessing as mp
|
|
||||||
+from concurrent.futures import ProcessPoolExecutor
|
|
||||||
+from concurrent.futures import TimeoutError
|
|
||||||
from functools import partial
|
|
||||||
-from multiprocessing.context import TimeoutError
|
|
||||||
from os.path import join, isfile, isdir, splitext
|
|
||||||
|
|
||||||
import padatious
|
|
||||||
@@ -95,15 +95,18 @@ def train(self, debug=True, single_thread=False, timeout=20):
|
|
||||||
train(i)
|
|
||||||
else:
|
|
||||||
# Train in multiple processes to disk
|
|
||||||
- pool = mp.Pool()
|
|
||||||
+ pool = ProcessPoolExecutor()
|
|
||||||
try:
|
|
||||||
- pool.map_async(train, self.objects_to_train).get(timeout)
|
|
||||||
+ _ = list(pool.map(train,
|
|
||||||
+ self.objects_to_train,
|
|
||||||
+ timeout=timeout))
|
|
||||||
+
|
|
||||||
except TimeoutError:
|
|
||||||
if debug:
|
|
||||||
print('Some objects timed out while training')
|
|
||||||
finally:
|
|
||||||
- pool.close()
|
|
||||||
- pool.join()
|
|
||||||
+ pass
|
|
||||||
+ # No explicit shutdown, let the it complete in the background.
|
|
||||||
|
|
||||||
# Load saved objects from disk
|
|
||||||
for obj in self.objects_to_train:
|
|
||||||
|
|
||||||
From f56b2944d1d470e28b089e814cdaab388619926e Mon Sep 17 00:00:00 2001
|
|
||||||
From: =?UTF-8?q?=C3=85ke=20Forslund?= <ake.forslund@gmail.com>
|
|
||||||
Date: Sat, 3 Apr 2021 11:15:37 +0200
|
|
||||||
Subject: [PATCH 2/2] Add proper subprocess timeout
|
|
||||||
|
|
||||||
---
|
|
||||||
padatious/intent_container.py | 18 +++++++++++-------
|
|
||||||
1 file changed, 11 insertions(+), 7 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/padatious/intent_container.py b/padatious/intent_container.py
|
|
||||||
index b020bf7..8a5544d 100644
|
|
||||||
--- a/padatious/intent_container.py
|
|
||||||
+++ b/padatious/intent_container.py
|
|
||||||
@@ -18,7 +18,7 @@
|
|
||||||
import padaos
|
|
||||||
import sys
|
|
||||||
from functools import wraps
|
|
||||||
-from subprocess import call, check_output
|
|
||||||
+from subprocess import call, check_output, TimeoutExpired
|
|
||||||
from threading import Thread
|
|
||||||
|
|
||||||
from padatious.match_data import MatchData
|
|
||||||
@@ -261,12 +261,16 @@ def train_subprocess(self, *args, **kwargs):
|
|
||||||
Returns:
|
|
||||||
bool: True for success, False if timed out
|
|
||||||
"""
|
|
||||||
- ret = call([
|
|
||||||
- sys.executable, '-m', 'padatious', 'train', self.cache_dir,
|
|
||||||
- '-d', json.dumps(self.serialized_args),
|
|
||||||
- '-a', json.dumps(args),
|
|
||||||
- '-k', json.dumps(kwargs),
|
|
||||||
- ])
|
|
||||||
+ try:
|
|
||||||
+ ret = call([
|
|
||||||
+ sys.executable, '-m', 'padatious', 'train', self.cache_dir,
|
|
||||||
+ '-d', json.dumps(self.serialized_args),
|
|
||||||
+ '-a', json.dumps(args),
|
|
||||||
+ '-k', json.dumps(kwargs),
|
|
||||||
+ ], timeout=kwargs.get('timeout'))
|
|
||||||
+ except TimeoutExpired:
|
|
||||||
+ ret = 10 # Treat process timeout as internal timeout
|
|
||||||
+
|
|
||||||
if ret == 2:
|
|
||||||
raise TypeError(
|
|
||||||
'Invalid train arguments: {} {}'.format(
|
|
||||||
Loading…
Reference in New Issue