[feature] support nested configuration files, and setting ALL configuration variables by CLI and env (#4109)

This updates our configuration code generator to now also include map marshal and unmarshalers. So we now have much more control over how things get read from pflags, and stored / read from viper configuration. This allows us to set ALL configuration variables by CLI and environment now, AND support nested configuration files. e.g.

```yaml
advanced:
    scraper-deterrence = true

http-client:
    allow-ips = ["127.0.0.1"]
```

is the same as

```yaml
advanced-scraper-deterrence = true

http-client-allow-ips = ["127.0.0.1"]
```

This also starts cleaning up of our jumbled Configuration{} type by moving the advanced configuration options into their own nested structs, also as a way to show what it's capable of. It's worth noting however that nesting only works if the Go types are nested too (as this is how we hint to our code generator to generate the necessary flattening code :p).

closes #3195

Reviewed-on: https://codeberg.org/superseriousbusiness/gotosocial/pulls/4109
Co-authored-by: kim <grufwub@gmail.com>
Co-committed-by: kim <grufwub@gmail.com>

vendor/codeberg.org/gruf/go-split/LICENSE

@@ -0,0 +1,9 @@
+MIT License
+
+Copyright (c) gruf
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

vendor/codeberg.org/gruf/go-split/README.md

@@ -0,0 +1,3 @@
+# go-split
+
+Performant string splitting and joining by comma, taking quotes into account. Useful for user supplied input e.g. CLI args

vendor/codeberg.org/gruf/go-split/join.go

@@ -0,0 +1,112 @@
+package split
+
+import (
+	"strconv"
+	"time"
+	"unsafe"
+
+	"codeberg.org/gruf/go-bytesize"
+)
+
+// joinFunc will join given slice of elements, using the passed function to append each element at index
+// from the slice, forming a combined comma-space separated string. Passed size is for buffer preallocation.
+func JoinFunc[T any](slice []T, each func(buf []byte, value T) []byte, size int) string {
+
+	// Move nil check
+	// outside main loop.
+	if each == nil {
+		panic("nil func")
+	}
+
+	// Catch easiest case
+	if len(slice) == 0 {
+		return ""
+	}
+
+	// Preallocate string buffer (size + commas)
+	buf := make([]byte, 0, size+len(slice)-1)
+
+	for _, value := range slice {
+		// Append each item
+		buf = each(buf, value)
+		buf = append(buf, ',', ' ')
+	}
+
+	// Drop final comma-space
+	buf = buf[:len(buf)-2]
+
+	// Directly cast buf to string
+	data := unsafe.SliceData(buf)
+	return unsafe.String(data, len(buf))
+}
+
+// JoinStrings will pass string slice to JoinFunc(), quoting where
+// necessary and combining into a single comma-space separated string.
+func JoinStrings[String ~string](slice []String) string {
+	var size int
+	for _, str := range slice {
+		size += len(str)
+	}
+	return JoinFunc(slice, func(buf []byte, value String) []byte {
+		return appendQuote(buf, string(value))
+	}, size)
+}
+
+// JoinBools will pass bool slice to JoinFunc(), formatting
+// and combining into a single comma-space separated string.
+func JoinBools[Bool ~bool](slice []Bool) string {
+	return JoinFunc(slice, func(buf []byte, value Bool) []byte {
+		return strconv.AppendBool(buf, bool(value))
+	}, len(slice)*5 /* len("false") */)
+}
+
+// JoinInts will pass signed integer slice to JoinFunc(), formatting
+// and combining into a single comma-space separated string.
+func JoinInts[Int Signed](slice []Int) string {
+	return JoinFunc(slice, func(buf []byte, value Int) []byte {
+		return strconv.AppendInt(buf, int64(value), 10)
+	}, len(slice)*20) // max signed int str len
+}
+
+// JoinUints will pass unsigned integer slice to JoinFunc(),
+// formatting and combining into a single comma-space separated string.
+func JoinUints[Uint Unsigned](slice []Uint) string {
+	return JoinFunc(slice, func(buf []byte, value Uint) []byte {
+		return strconv.AppendUint(buf, uint64(value), 10)
+	}, len(slice)*20) // max unsigned int str len
+}
+
+// JoinFloats will pass float slice to JoinFunc(), formatting
+// and combining into a single comma-space separated string.
+func JoinFloats[Float_ Float](slice []Float_) string {
+	bits := int(unsafe.Sizeof(Float_(0)) * 8) // param type bits
+	return JoinFunc(slice, func(buf []byte, value Float_) []byte {
+		return strconv.AppendFloat(buf, float64(value), 'g', -1, bits)
+	}, len(slice)*20) // max signed int str len (it's a good guesstimate)
+}
+
+// JoinSizes will pass byte size slice to JoinFunc(), formatting
+// and combining into a single comma-space separated string.
+func JoinSizes(slice []bytesize.Size) string {
+	const iecLen = 7 // max IEC string length
+	return JoinFunc(slice, func(buf []byte, value bytesize.Size) []byte {
+		return value.AppendFormatIEC(buf)
+	}, len(slice)*iecLen)
+}
+
+// JoinDurations will pass duration slice to JoinFunc(), formatting
+// and combining into a single comma-space separated string.
+func JoinDurations(slice []time.Duration) string {
+	const durLen = 10 // max duration string length
+	return JoinFunc(slice, func(buf []byte, value time.Duration) []byte {
+		return append(buf, value.String()...)
+	}, len(slice)*durLen)
+}
+
+// JoinTimes will pass time slice to JoinFunc(), formatting
+// and combining into a single comma-space separated string.
+func JoinTimes(slice []time.Time, format string) string {
+	return JoinFunc(slice, func(buf []byte, value time.Time) []byte {
+		return value.AppendFormat(buf, format)
+	}, len(slice)*len(format))
+}

vendor/codeberg.org/gruf/go-split/join_util.go

@@ -0,0 +1,71 @@
+package split
+
+import (
+	"strconv"
+	"strings"
+)
+
+// singleTermLine: beyond a certain length of string, all of the
+// extra checks to handle quoting/not-quoting add a significant
+// amount of extra processing time. Quoting in this manner only really
+// effects readability on a single line, so a max string length that
+// encompasses the maximum number of columns on *most* terminals was
+// selected. This was chosen using the metric that 1080p is one of the
+// most common display resolutions, and that a relatively small font size
+// of 7 requires ~ 223 columns. So 256 should be >= $COLUMNS (fullscreen)
+// in 99% of usecases (these figures all pulled out of my ass).
+const singleTermLine = 256
+
+// appendQuote will append 'str' to 'buf', double quoting and escaping if needed.
+func appendQuote(buf []byte, str string) []byte {
+	switch {
+	case len(str) > singleTermLine || !strconv.CanBackquote(str):
+		// Append quoted and escaped string
+		return strconv.AppendQuote(buf, str)
+
+	case (strings.IndexByte(str, '"') != -1):
+		// Double quote and escape string
+		buf = append(buf, '"')
+		buf = appendEscape(buf, str)
+		buf = append(buf, '"')
+		return buf
+
+	case (strings.IndexByte(str, ',') != -1):
+		// Double quote this string as-is
+		buf = append(buf, '"')
+		buf = append(buf, str...)
+		buf = append(buf, '"')
+		return buf
+
+	default:
+		// Append string as-is
+		return append(buf, str...)
+	}
+}
+
+// appendEscape will append 'str' to 'buf' and escape any double quotes.
+func appendEscape(buf []byte, str string) []byte {
+	var delim bool
+	for i := range str {
+		switch {
+		case str[i] == '\\' && !delim:
+			// Set delim flag
+			delim = true
+
+		case str[i] == '"' && !delim:
+			// Append escaped double quote
+			buf = append(buf, `\"`...)
+
+		case delim:
+			// Append skipped slash
+			buf = append(buf, `\`...)
+			delim = false
+			fallthrough
+
+		default:
+			// Append char as-is
+			buf = append(buf, str[i])
+		}
+	}
+	return buf
+}

vendor/codeberg.org/gruf/go-split/split.go

@@ -0,0 +1,187 @@
+package split
+
+import (
+	"strconv"
+	"time"
+	"unsafe"
+
+	"codeberg.org/gruf/go-bytesize"
+)
+
+// Signed defines a signed
+// integer generic type parameter.
+type Signed interface {
+	~int | ~int8 | ~int16 | ~int32 | ~int64
+}
+
+// Unsigned defines an unsigned
+// integer generic type paramter.
+type Unsigned interface {
+	~uint | ~uint8 | ~uint16 | ~uint32 | ~uint64
+}
+
+// Float defines a float-type generic parameter.
+type Float interface{ ~float32 | ~float64 }
+
+// SplitFunc will split input string on commas, taking into account string quoting
+// and stripping extra whitespace, passing each split to the given function hook.
+func SplitFunc(str string, fn func(string) error) error {
+	return (&Splitter{}).SplitFunc(str, fn)
+}
+
+// SplitStrings will pass string input to SplitFunc(), compiling a slice of strings.
+func SplitStrings[String ~string](str string) ([]String, error) {
+	var slice []String
+
+	// Simply append each split string to slice
+	if err := SplitFunc(str, func(s string) error {
+		slice = append(slice, String(s))
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
+	return slice, nil
+}
+
+// SplitBools will pass string input to SplitFunc(), parsing and compiling a slice of bools.
+func SplitBools[Bool ~bool](str string) ([]Bool, error) {
+	var slice []Bool
+
+	// Parse each bool split from input string
+	if err := SplitFunc(str, func(s string) error {
+		b, err := strconv.ParseBool(s)
+		if err != nil {
+			return err
+		}
+		slice = append(slice, Bool(b))
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
+	return slice, nil
+}
+
+// SplitInts will pass string input to SplitFunc(), parsing and compiling a slice of signed integers.
+func SplitInts[Int Signed](str string) ([]Int, error) {
+	// Determine bits from param type size
+	bits := int(unsafe.Sizeof(Int(0)) * 8)
+
+	var slice []Int
+
+	// Parse each int split from input string
+	if err := SplitFunc(str, func(s string) error {
+		i, err := strconv.ParseInt(s, 10, bits)
+		if err != nil {
+			return err
+		}
+		slice = append(slice, Int(i))
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
+	return slice, nil
+}
+
+// SplitUints will pass string input to SplitFunc(), parsing and compiling a slice of unsigned integers.
+func SplitUints[Uint Unsigned](str string) ([]Uint, error) {
+	// Determine bits from param type size
+	bits := int(unsafe.Sizeof(Uint(0)) * 8)
+
+	var slice []Uint
+
+	// Parse each uint split from input string
+	if err := SplitFunc(str, func(s string) error {
+		u, err := strconv.ParseUint(s, 10, bits)
+		if err != nil {
+			return err
+		}
+		slice = append(slice, Uint(u))
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
+	return slice, nil
+}
+
+// SplitFloats will pass string input to SplitFunc(), parsing and compiling a slice of floats.
+func SplitFloats[Float_ Float](str string) ([]Float_, error) {
+	// Determine bits from param type size
+	bits := int(unsafe.Sizeof(Float_(0)) * 8)
+
+	var slice []Float_
+
+	// Parse each float split from input string
+	if err := SplitFunc(str, func(s string) error {
+		f, err := strconv.ParseFloat(s, bits)
+		if err != nil {
+			return err
+		}
+		slice = append(slice, Float_(f))
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
+	return slice, nil
+}
+
+// SplitSizes will pass string input to SplitFunc(), parsing and compiling a slice of byte sizes.
+func SplitSizes(str string) ([]bytesize.Size, error) {
+	var slice []bytesize.Size
+
+	// Parse each size split from input string
+	if err := SplitFunc(str, func(s string) error {
+		sz, err := bytesize.ParseSize(s)
+		if err != nil {
+			return err
+		}
+		slice = append(slice, sz)
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
+	return slice, nil
+}
+
+// SplitDurations will pass string input to SplitFunc(), parsing and compiling a slice of durations.
+func SplitDurations(str string) ([]time.Duration, error) {
+	var slice []time.Duration
+
+	// Parse each duration split from input string
+	if err := SplitFunc(str, func(s string) error {
+		d, err := time.ParseDuration(s)
+		if err != nil {
+			return err
+		}
+		slice = append(slice, d)
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
+	return slice, nil
+}
+
+// SplitTimes will pass string input to SplitFunc(), parsing and compiling a slice of times.
+func SplitTimes(str string, format string) ([]time.Time, error) {
+	var slice []time.Time
+
+	// Parse each time split from input string
+	if err := SplitFunc(str, func(s string) error {
+		t, err := time.Parse(s, format)
+		if err != nil {
+			return err
+		}
+		slice = append(slice, t)
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
+	return slice, nil
+}

vendor/codeberg.org/gruf/go-split/splitter.go

@@ -0,0 +1,204 @@
+package split
+
+import (
+	"errors"
+	"strings"
+	"unicode"
+	"unicode/utf8"
+)
+
+// Splitter holds onto a byte buffer for use in minimising allocations during SplitFunc().
+type Splitter struct{ B []byte }
+
+// SplitFunc will split input string on commas, taking into account string quoting and
+// stripping extra whitespace, passing each split to the given function hook.
+func (s *Splitter) SplitFunc(str string, fn func(string) error) error {
+	for {
+		// Reset buffer
+		s.B = s.B[0:0]
+
+		// Trim leading space
+		str = trimLeadingSpace(str)
+
+		if len(str) < 1 {
+			// Reached end
+			return nil
+		}
+
+		switch {
+		// Single / double quoted
+		case str[0] == '\'', str[0] == '"':
+			// Calculate next string elem
+			i := 1 + s.next(str[1:], str[0])
+			if i == 0 /* i.e. if .next() returned -1 */ {
+				return errors.New("missing end quote")
+			}
+
+			// Pass next element to callback func
+			if err := fn(string(s.B)); err != nil {
+				return err
+			}
+
+			// Reslice + trim leading space
+			str = trimLeadingSpace(str[i+1:])
+
+			if len(str) < 1 {
+				// reached end
+				return nil
+			}
+
+			if str[0] != ',' {
+				// malformed element without comma after quote
+				return errors.New("missing comma separator")
+			}
+
+			// Skip comma
+			str = str[1:]
+
+		// Empty segment
+		case str[0] == ',':
+			str = str[1:]
+
+		// No quoting
+		default:
+			// Calculate next string elem
+			i := s.next(str, ',')
+
+			switch i {
+			// Reached end
+			case -1:
+				// we know len > 0
+
+				// Pass to callback
+				return fn(string(s.B))
+
+			// Empty elem
+			case 0:
+				str = str[1:]
+
+			// Non-zero elem
+			default:
+				// Pass next element to callback
+				if err := fn(string(s.B)); err != nil {
+					return err
+				}
+
+				// Skip past eleme
+				str = str[i+1:]
+			}
+		}
+	}
+}
+
+// next will build the next string element in s.B up to non-delimited instance of c,
+// returning number of characters iterated, or -1 if the end of the string was reached.
+func (s *Splitter) next(str string, c byte) int {
+	var delims int
+
+	// Guarantee buf large enough
+	if len(str) > cap(s.B)-len(s.B) {
+		nb := make([]byte, 2*cap(s.B)+len(str))
+		_ = copy(nb, s.B)
+		s.B = nb[:len(s.B)]
+	}
+
+	for i := 0; i < len(str); i++ {
+		// Increment delims
+		if str[i] == '\\' {
+			delims++
+			continue
+		}
+
+		if str[i] == c {
+			var count int
+
+			if count = delims / 2; count > 0 {
+				// Add backslashes to buffer
+				slashes := backslashes(count)
+				s.B = append(s.B, slashes...)
+			}
+
+			// Reached delim'd char
+			if delims-count == 0 {
+				return i
+			}
+		} else if delims > 0 {
+			// Add backslashes to buffer
+			slashes := backslashes(delims)
+			s.B = append(s.B, slashes...)
+		}
+
+		// Write byte to buffer
+		s.B = append(s.B, str[i])
+
+		// Reset count
+		delims = 0
+	}
+
+	return -1
+}
+
+// asciiSpace is a lookup table of ascii space chars (see: strings.asciiSet).
+var asciiSpace = func() (as [8]uint32) {
+	as['\t'/32] |= 1 << ('\t' % 32)
+	as['\n'/32] |= 1 << ('\n' % 32)
+	as['\v'/32] |= 1 << ('\v' % 32)
+	as['\f'/32] |= 1 << ('\f' % 32)
+	as['\r'/32] |= 1 << ('\r' % 32)
+	as[' '/32] |= 1 << (' ' % 32)
+	return
+}()
+
+// trimLeadingSpace trims the leading space from a string.
+func trimLeadingSpace(str string) string {
+	var start int
+
+	for ; start < len(str); start++ {
+		// If beyond ascii range, trim using slower rune check.
+		if str[start] >= utf8.RuneSelf {
+			return trimLeadingSpaceSlow(str[start:])
+		}
+
+		// Ascii character
+		char := str[start]
+
+		// This is first non-space ASCII, trim up to here
+		if (asciiSpace[char/32] & (1 << (char % 32))) == 0 {
+			break
+		}
+	}
+
+	return str[start:]
+}
+
+// trimLeadingSpaceSlow trims leading space using the slower unicode.IsSpace check.
+func trimLeadingSpaceSlow(str string) string {
+	for i, r := range str {
+		if !unicode.IsSpace(r) {
+			return str[i:]
+		}
+	}
+	return str
+}
+
+// backslashes will return a string of backslashes of given length.
+func backslashes(count int) string {
+	const backslashes = `\\\\\\\\\\\\\\\\\\\\`
+
+	// Fast-path, use string const
+	if count < len(backslashes) {
+		return backslashes[:count]
+	}
+
+	// Slow-path, build custom string
+	return backslashSlow(count)
+}
+
+// backslashSlow will build a string of backslashes of custom length.
+func backslashSlow(count int) string {
+	var buf strings.Builder
+	for i := 0; i < count; i++ {
+		buf.WriteByte('\\')
+	}
+	return buf.String()
+}