SuperSeriousBusiness/GoToSocial
1
1
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial synced 2025-06-05 21:59:39 +02:00
Code Issues Projects Releases Wiki Activity

[chore]: Bump github.com/KimMachineGun/automemlimit from 0.4.0 to 0.5.0 (#2560)

Browse Source
This commit is contained in:
dependabot[bot]
2024-01-22 09:35:23 +00:00
committed by GitHub
parent 4e0488acfe
commit a858831387
18 changed files with 513 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
vendor/github.com/KimMachineGun/automemlimit/README.md generated vendored
View File

@ -8,6 +8,14 @@ Automatically set `GOMEMLIMIT` to match Linux [cgroups(7)](https://man7.org/linu
See more details about `GOMEMLIMIT` [here](https://tip.golang.org/doc/gc-guide#Memory_limit).
## Notice
Version `v0.5.0` introduces a fallback to system memory limits as an experimental feature when cgroup limits are unavailable. Activate this by setting `AUTOMEMLIMIT_EXPERIMENT=system`.
You can also use system memory limits via `memlimit.FromSystem` provider directly.
This feature is under evaluation and might become a default or be removed based on user feedback.
If you have any feedback about this feature, please open an issue.
## Installation
```shell
@ -34,9 +42,17 @@ import "github.com/KimMachineGun/automemlimit/memlimit"
func init() {
memlimit.SetGoMemLimitWithOpts(
memlimit.WithRatio(0.9),
memlimit.WithEnv(),
memlimit.WithProvider(memlimit.FromCgroup),
)
memlimit.SetGoMemLimitWithOpts(
memlimit.WithRatio(0.9),
memlimit.WithProvider(
memlimit.ApplyFallback(
memlimit.FromCgroup,
memlimit.FromSystem,
),
),
)
memlimit.SetGoMemLimitWithEnv()
memlimit.SetGoMemLimit(0.9)
memlimit.SetGoMemLimitWithProvider(memlimit.Limit(1024*1024), 0.9)

91
vendor/github.com/KimMachineGun/automemlimit/memlimit/cgroups.go generated vendored
View File

@ -1,91 +1,12 @@
//go:build linux
// +build linux
package memlimit
import (
"path/filepath"
"github.com/containerd/cgroups/v3"
"github.com/containerd/cgroups/v3/cgroup1"
"github.com/containerd/cgroups/v3/cgroup2"
"errors"
)
const (
cgroupMountPoint = "/sys/fs/cgroup"
var (
// ErrNoCgroup is returned when the process is not in cgroup.
ErrNoCgroup = errors.New("process is not in cgroup")
// ErrCgroupsNotSupported is returned when the system does not support cgroups.
ErrCgroupsNotSupported = errors.New("cgroups is not supported on this system")
)
// FromCgroup returns the memory limit based on the cgroups version on this system.
func FromCgroup() (uint64, error) {
switch cgroups.Mode() {
case cgroups.Legacy:
return FromCgroupV1()
case cgroups.Hybrid:
return FromCgroupHybrid()
case cgroups.Unified:
return FromCgroupV2()
}
return 0, ErrNoCgroup
}
// FromCgroupV1 returns the memory limit from the cgroup v1.
func FromCgroupV1() (uint64, error) {
cg, err := cgroup1.Load(cgroup1.RootPath, cgroup1.WithHiearchy(
cgroup1.SingleSubsystem(cgroup1.Default, cgroup1.Memory),
))
if err != nil {
return 0, err
}
metrics, err := cg.Stat(cgroup1.IgnoreNotExist)
if err != nil {
return 0, err
}
if limit := metrics.GetMemory().GetHierarchicalMemoryLimit(); limit != 0 {
return limit, nil
}
return 0, ErrNoLimit
}
// FromCgroupHybrid returns the memory limit from the cgroup v1 or v2.
// It checks the cgroup v2 first, and if it fails, it falls back to cgroup v1.
func FromCgroupHybrid() (uint64, error) {
limit, err := fromCgroupV2(filepath.Join(cgroupMountPoint, "unified"))
if err == nil {
return limit, nil
} else if err != ErrNoLimit {
return 0, err
}
return FromCgroupV1()
}
// FromCgroupV2 returns the memory limit from the cgroup v2.
func FromCgroupV2() (uint64, error) {
return fromCgroupV2(cgroupMountPoint)
}
func fromCgroupV2(mountPoint string) (uint64, error) {
path, err := cgroup2.NestedGroupPath("")
if err != nil {
return 0, err
}
m, err := cgroup2.Load(path, cgroup2.WithMountpoint(mountPoint))
if err != nil {
return 0, err
}
stats, err := m.Stat()
if err != nil {
return 0, err
}
if limit := stats.GetMemory().GetUsageLimit(); limit != 0 {
return limit, nil
}
return 0, ErrNoLimit
}

98
vendor/github.com/KimMachineGun/automemlimit/memlimit/cgroups_linux.go generated vendored Normal file
View File

@ -0,0 +1,98 @@
//go:build linux
// +build linux
package memlimit
import (
"math"
"os"
"path/filepath"
"github.com/containerd/cgroups/v3"
"github.com/containerd/cgroups/v3/cgroup1"
"github.com/containerd/cgroups/v3/cgroup2"
)
const (
cgroupMountPoint = "/sys/fs/cgroup"
)
// FromCgroup returns the memory limit based on the cgroups version on this system.
func FromCgroup() (uint64, error) {
switch cgroups.Mode() {
case cgroups.Legacy:
return FromCgroupV1()
case cgroups.Hybrid:
return FromCgroupHybrid()
case cgroups.Unified:
return FromCgroupV2()
}
return 0, ErrNoCgroup
}
// FromCgroupV1 returns the memory limit from the cgroup v1.
func FromCgroupV1() (uint64, error) {
cg, err := cgroup1.Load(cgroup1.RootPath, cgroup1.WithHiearchy(
cgroup1.SingleSubsystem(cgroup1.Default, cgroup1.Memory),
))
if err != nil {
return 0, err
}
metrics, err := cg.Stat(cgroup1.IgnoreNotExist)
if err != nil {
return 0, err
}
if limit := metrics.GetMemory().GetHierarchicalMemoryLimit(); limit != 0 && limit != getCgroupV1NoLimit() {
return limit, nil
}
return 0, ErrNoLimit
}
func getCgroupV1NoLimit() uint64 {
ps := uint64(os.Getpagesize())
return math.MaxInt64 / ps * ps
}
// FromCgroupHybrid returns the memory limit from the cgroup v1 or v2.
// It checks the cgroup v2 first, and if it fails, it falls back to cgroup v1.
func FromCgroupHybrid() (uint64, error) {
limit, err := fromCgroupV2(filepath.Join(cgroupMountPoint, "unified"))
if err == nil {
return limit, nil
} else if err != ErrNoLimit {
return 0, err
}
return FromCgroupV1()
}
// FromCgroupV2 returns the memory limit from the cgroup v2.
func FromCgroupV2() (uint64, error) {
return fromCgroupV2(cgroupMountPoint)
}
func fromCgroupV2(mountPoint string) (uint64, error) {
path, err := cgroup2.NestedGroupPath("")
if err != nil {
return 0, err
}
m, err := cgroup2.Load(path, cgroup2.WithMountpoint(mountPoint))
if err != nil {
return 0, err
}
stats, err := m.Stat()
if err != nil {
return 0, err
}
if limit := stats.GetMemory().GetUsageLimit(); limit != 0 && limit != math.MaxUint64 {
return limit, nil
}
return 0, ErrNoLimit
}

14
vendor/github.com/KimMachineGun/automemlimit/memlimit/exp_system.go generated vendored Normal file
View File

@ -0,0 +1,14 @@
package memlimit
import (
"github.com/pbnjay/memory"
)
// FromSystem returns the total memory of the system.
func FromSystem() (uint64, error) {
limit := memory.TotalMemory()
if limit == 0 {
return 0, ErrNoLimit
}
return limit, nil
}

59
vendor/github.com/KimMachineGun/automemlimit/memlimit/experiment.go generated vendored Normal file
View File

@ -0,0 +1,59 @@
package memlimit
import (
"fmt"
"os"
"reflect"
"strings"
)
const (
envAUTOMEMLIMIT_EXPERIMENT = "AUTOMEMLIMIT_EXPERIMENT"
)
// Experiments is a set of experiment flags.
// It is used to enable experimental features.
//
// You can set the flags by setting the environment variable AUTOMEMLIMIT_EXPERIMENT.
// The value of the environment variable is a comma-separated list of experiment names.
//
// The following experiment names are known:
//
// - none: disable all experiments
// - system: enable fallback to system memory limit
type Experiments struct {
// System enables fallback to system memory limit.
System bool
}
func parseExperiments() (Experiments, error) {
var exp Experiments
// Create a map of known experiment names.
names := make(map[string]func(bool))
rv := reflect.ValueOf(&exp).Elem()
rt := rv.Type()
for i := 0; i < rt.NumField(); i++ {
field := rv.Field(i)
names[strings.ToLower(rt.Field(i).Name)] = field.SetBool
}
// Parse names.
for _, f := range strings.Split(os.Getenv(envAUTOMEMLIMIT_EXPERIMENT), ",") {
if f == "" {
continue
}
if f == "none" {
exp = Experiments{}
continue
}
val := true
set, ok := names[f]
if !ok {
return Experiments{}, fmt.Errorf("unknown AUTOMEMLIMIT_EXPERIMENT %s", f)
}
set(val)
}
return exp, nil
}

45
vendor/github.com/KimMachineGun/automemlimit/memlimit/memlimit.go generated vendored
View File

@ -22,16 +22,11 @@ const (
var (
// ErrNoLimit is returned when the memory limit is not set.
ErrNoLimit = errors.New("memory is not limited")
// ErrNoCgroup is returned when the process is not in cgroup.
ErrNoCgroup = errors.New("process is not in cgroup")
// ErrCgroupsNotSupported is returned when the system does not support cgroups.
ErrCgroupsNotSupported = errors.New("cgroups is not supported on this system")
)
type config struct {
logger *log.Logger
ratio float64
env bool
provider Provider
}
@ -50,10 +45,10 @@ func WithRatio(ratio float64) Option {
// WithEnv configures whether to use environment variables.
//
// Default: false
//
// Deprecated: currently this does nothing.
func WithEnv() Option {
return func(cfg *config) {
cfg.env = true
}
return func(cfg *config) {}
}
// WithProvider configures the provider.
@ -65,17 +60,24 @@ func WithProvider(provider Provider) Option {
}
}
// SetGoMemLimitWithOpts sets GOMEMLIMIT with options.
// SetGoMemLimitWithOpts sets GOMEMLIMIT with options and environment variables.
//
// You can configure how much memory of the cgroup's memory limit to set as GOMEMLIMIT
// through AUTOMEMLIMIT envrironment variable in the half-open range (0.0,1.0].
//
// If AUTOMEMLIMIT is not set, it defaults to 0.9. (10% is the headroom for memory sources the Go runtime is unaware of.)
// If GOMEMLIMIT is already set or AUTOMEMLIMIT=off, this function does nothing.
//
// If AUTOMEMLIMIT_EXPERIMENT is set, it enables experimental features.
// Please see the documentation of Experiments for more details.
//
// Options:
// - WithRatio
// - WithEnv (see more SetGoMemLimitWithEnv)
// - WithProvider
func SetGoMemLimitWithOpts(opts ...Option) (_ int64, _err error) {
cfg := &config{
logger: log.New(io.Discard, "", log.LstdFlags),
ratio: defaultAUTOMEMLIMIT,
env: false,
provider: FromCgroup,
}
if os.Getenv(envAUTOMEMLIMIT_DEBUG) == "true" {
@ -90,6 +92,15 @@ func SetGoMemLimitWithOpts(opts ...Option) (_ int64, _err error) {
}
}()
exps, err := parseExperiments()
if err != nil {
return 0, fmt.Errorf("failed to parse experiments: %w", err)
}
if exps.System {
cfg.logger.Println("system experiment is enabled: using system memory limit as a fallback")
cfg.provider = ApplyFallback(cfg.provider, FromSystem)
}
snapshot := debug.SetMemoryLimit(-1)
defer func() {
err := recover()
@ -122,6 +133,10 @@ func SetGoMemLimitWithOpts(opts ...Option) (_ int64, _err error) {
limit, err := setGoMemLimit(ApplyRatio(cfg.provider, ratio))
if err != nil {
if errors.Is(err, ErrNoLimit) {
cfg.logger.Printf("memory is not limited, skipping: %v\n", err)
return 0, nil
}
return 0, fmt.Errorf("failed to set GOMEMLIMIT: %w", err)
}
@ -130,14 +145,8 @@ func SetGoMemLimitWithOpts(opts ...Option) (_ int64, _err error) {
return limit, nil
}
// SetGoMemLimitWithEnv sets GOMEMLIMIT with the value from the environment variable.
// You can configure how much memory of the cgroup's memory limit to set as GOMEMLIMIT
// through AUTOMEMLIMIT in the half-open range (0.0,1.0].
//
// If AUTOMEMLIMIT is not set, it defaults to 0.9. (10% is the headroom for memory sources the Go runtime is unaware of.)
// If GOMEMLIMIT is already set or AUTOMEMLIMIT=off, this function does nothing.
func SetGoMemLimitWithEnv() {
_, _ = SetGoMemLimitWithOpts(WithEnv())
_, _ = SetGoMemLimitWithOpts()
}
// SetGoMemLimit sets GOMEMLIMIT with the value from the cgroup's memory limit and given ratio.