2024-08-05 18:21:25 +02:00
|
|
|
diff --git third_party/sentencepiece/src/src/util.cc third_party/sentencepiece/src/src/util.cc
|
2024-08-26 14:44:25 +02:00
|
|
|
index c5e5289807a0c..e10880abc17eb 100644
|
2024-08-05 18:21:25 +02:00
|
|
|
--- third_party/sentencepiece/src/src/util.cc
|
|
|
|
+++ third_party/sentencepiece/src/src/util.cc
|
|
|
|
@@ -16,6 +16,7 @@
|
|
|
|
|
|
|
|
#include <atomic>
|
|
|
|
#include <iostream>
|
|
|
|
+#include <memory>
|
|
|
|
|
|
|
|
namespace sentencepiece {
|
|
|
|
|
2024-08-26 14:44:25 +02:00
|
|
|
@@ -197,8 +198,18 @@ std::mt19937 *GetRandomGenerator() {
|
2024-08-05 18:21:25 +02:00
|
|
|
}
|
|
|
|
#else
|
|
|
|
std::mt19937 *GetRandomGenerator() {
|
|
|
|
- thread_local static std::mt19937 mt(GetRandomGeneratorSeed());
|
|
|
|
- return &mt;
|
|
|
|
+ // Thread-locals occupy stack space in every thread ever created by the
|
|
|
|
+ // program, even if that thread never uses the thread-local variable.
|
|
|
|
+ //
|
|
|
|
+ // https://maskray.me/blog/2021-02-14-all-about-thread-local-storage
|
|
|
|
+ //
|
|
|
|
+ // sizeof(std::mt19937) is several kilobytes, so it is safer to put that on
|
|
|
|
+ // the heap, leaving only a pointer to it in thread-local storage. This must
|
|
|
|
+ // be a unique_ptr, not a raw pointer, so that the generator is not leaked on
|
|
|
|
+ // thread exit.
|
|
|
|
+ thread_local static auto mt =
|
|
|
|
+ std::make_unique<std::mt19937>(GetRandomGeneratorSeed());
|
|
|
|
+ return mt.get();
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
} // namespace random
|