34 lines
1.1 KiB
Diff
34 lines
1.1 KiB
Diff
|
diff --git third_party/sentencepiece/src/src/util.cc third_party/sentencepiece/src/src/util.cc
|
||
|
|
index 538b00b..61c4e5d 100644
|
||
|
|
--- third_party/sentencepiece/src/src/util.cc
|
||
|
|
+++ third_party/sentencepiece/src/src/util.cc
|
||
|
|
@@ -16,6 +16,7 @@
|
||
|
|
|
||
|
|
#include <atomic>
|
||
|
|
#include <iostream>
|
||
|
|
+#include <memory>
|
||
|
|
|
||
|
|
namespace sentencepiece {
|
||
|
|
|
||
|
|
@@ -187,8 +190,18 @@ std::mt19937 *GetRandomGenerator() {
|
||
|
|
}
|
||
|
|
#else
|
||
|
|
std::mt19937 *GetRandomGenerator() {
|
||
|
|
- thread_local static std::mt19937 mt(GetRandomGeneratorSeed());
|
||
|
|
- return &mt;
|
||
|
|
+ // Thread-locals occupy stack space in every thread ever created by the
|
||
|
|
+ // program, even if that thread never uses the thread-local variable.
|
||
|
|
+ //
|
||
|
|
+ // https://maskray.me/blog/2021-02-14-all-about-thread-local-storage
|
||
|
|
+ //
|
||
|
|
+ // sizeof(std::mt19937) is several kilobytes, so it is safer to put that on
|
||
|
|
+ // the heap, leaving only a pointer to it in thread-local storage. This must
|
||
|
|
+ // be a unique_ptr, not a raw pointer, so that the generator is not leaked on
|
||
|
|
+ // thread exit.
|
||
|
|
+ thread_local static auto mt =
|
||
|
|
+ std::make_unique<std::mt19937>(GetRandomGeneratorSeed());
|
||
|
|
+ return mt.get();
|
||
|
|
}
|
||
|
|
#endif
|
||
|
|
} // namespace random
|