[feature] Tune sqlite pragmas (#1349)
* sqlite pragma tuning * use formatuint * add sqlite busy timeout * fix incorrect cache size format * update envparsing test * add sqlite tuning flags to cli * set sqlite timeout to 30s default
This commit is contained in:
tobi
GitHub
parent
a6c6bdb34a
commit
627b8eeae6
|
|
@ -107,4 +107,36 @@ db-tls-mode: "disable"
|
|||
# Examples: ["/path/to/some/cert.crt"]
|
||||
# Default: ""
|
||||
db-tls-ca-cert: ""
|
||||
|
||||
# String. SQLite journaling mode.
|
||||
# SQLite only -- unused otherwise.
|
||||
# If set to empty string, the sqlite default will be used.
|
||||
# See: https://www.sqlite.org/pragma.html#pragma_journal_mode
|
||||
# Examples: ["DELETE", "TRUNCATE", "PERSIST", "MEMORY", "WAL", "OFF"]
|
||||
# Default: "WAL"
|
||||
db-sqlite-journal-mode: "WAL"
|
||||
|
||||
# String. SQLite synchronous mode.
|
||||
# SQLite only -- unused otherwise.
|
||||
# If set to empty string, the sqlite default will be used.
|
||||
# See: https://www.sqlite.org/pragma.html#pragma_synchronous
|
||||
# Examples: ["OFF", "NORMAL", "FULL", "EXTRA"]
|
||||
# Default: "NORMAL"
|
||||
db-sqlite-synchronous: "NORMAL"
|
||||
|
||||
# Byte size. SQlite cache size.
|
||||
# SQLite only -- unused otherwise.
|
||||
# If set to empty string or zero, the sqlite default will be used.
|
||||
# See: https://www.sqlite.org/pragma.html#pragma_cache_size
|
||||
# Examples: ["32MiB", "0", "64MiB"]
|
||||
# Default: "64MiB"
|
||||
db-sqlite-cache-size: "64MiB"
|
||||
|
||||
# Duration. SQlite busy timeout.
|
||||
# SQLite only -- unused otherwise.
|
||||
# If set to empty string or zero, the sqlite default will be used.
|
||||
# See: https://www.sqlite.org/pragma.html#pragma_busy_timeout
|
||||
# Examples: ["0s", "1s", "30s", "1m", "5m"]
|
||||
# Default: "5s"
|
||||
db-sqlite-busy-timeout: "30s"
|
||||
```
|
||||
|
|
|
|||
|
|
@ -164,6 +164,38 @@ db-tls-mode: "disable"
|
|||
# Default: ""
|
||||
db-tls-ca-cert: ""
|
||||
|
||||
# String. SQLite journaling mode.
|
||||
# SQLite only -- unused otherwise.
|
||||
# If set to empty string, the sqlite default will be used.
|
||||
# See: https://www.sqlite.org/pragma.html#pragma_journal_mode
|
||||
# Examples: ["DELETE", "TRUNCATE", "PERSIST", "MEMORY", "WAL", "OFF"]
|
||||
# Default: "WAL"
|
||||
db-sqlite-journal-mode: "WAL"
|
||||
|
||||
# String. SQLite synchronous mode.
|
||||
# SQLite only -- unused otherwise.
|
||||
# If set to empty string, the sqlite default will be used.
|
||||
# See: https://www.sqlite.org/pragma.html#pragma_synchronous
|
||||
# Examples: ["OFF", "NORMAL", "FULL", "EXTRA"]
|
||||
# Default: "NORMAL"
|
||||
db-sqlite-synchronous: "NORMAL"
|
||||
|
||||
# Byte size. SQlite cache size.
|
||||
# SQLite only -- unused otherwise.
|
||||
# If set to empty string or zero, the sqlite default will be used.
|
||||
# See: https://www.sqlite.org/pragma.html#pragma_cache_size
|
||||
# Examples: ["32MiB", "0", "64MiB"]
|
||||
# Default: "64MiB"
|
||||
db-sqlite-cache-size: "64MiB"
|
||||
|
||||
# Duration. SQlite busy timeout.
|
||||
# SQLite only -- unused otherwise.
|
||||
# If set to empty string or zero, the sqlite default will be used.
|
||||
# See: https://www.sqlite.org/pragma.html#pragma_busy_timeout
|
||||
# Examples: ["0s", "1s", "30s", "1m", "5m"]
|
||||
# Default: "5s"
|
||||
db-sqlite-busy-timeout: "30s"
|
||||
|
||||
cache:
|
||||
gts:
|
||||
###########################
|
||||
|
|
|
|||
|
|
@ -58,14 +58,18 @@ type Configuration struct {
|
|||
TrustedProxies []string `name:"trusted-proxies" usage:"Proxies to trust when parsing x-forwarded headers into real IPs."`
|
||||
SoftwareVersion string `name:"software-version" usage:""`
|
||||
|
||||
DbType string `name:"db-type" usage:"Database type: eg., postgres"`
|
||||
DbAddress string `name:"db-address" usage:"Database ipv4 address, hostname, or filename"`
|
||||
DbPort int `name:"db-port" usage:"Database port"`
|
||||
DbUser string `name:"db-user" usage:"Database username"`
|
||||
DbPassword string `name:"db-password" usage:"Database password"`
|
||||
DbDatabase string `name:"db-database" usage:"Database name"`
|
||||
DbTLSMode string `name:"db-tls-mode" usage:"Database tls mode"`
|
||||
DbTLSCACert string `name:"db-tls-ca-cert" usage:"Path to CA cert for db tls connection"`
|
||||
DbType string `name:"db-type" usage:"Database type: eg., postgres"`
|
||||
DbAddress string `name:"db-address" usage:"Database ipv4 address, hostname, or filename"`
|
||||
DbPort int `name:"db-port" usage:"Database port"`
|
||||
DbUser string `name:"db-user" usage:"Database username"`
|
||||
DbPassword string `name:"db-password" usage:"Database password"`
|
||||
DbDatabase string `name:"db-database" usage:"Database name"`
|
||||
DbTLSMode string `name:"db-tls-mode" usage:"Database tls mode"`
|
||||
DbTLSCACert string `name:"db-tls-ca-cert" usage:"Path to CA cert for db tls connection"`
|
||||
DbSqliteJournalMode string `name:"db-sqlite-journal-mode" usage:"Sqlite only: see https://www.sqlite.org/pragma.html#pragma_journal_mode"`
|
||||
DbSqliteSynchronous string `name:"db-sqlite-synchronous" usage:"Sqlite only: see https://www.sqlite.org/pragma.html#pragma_synchronous"`
|
||||
DbSqliteCacheSize bytesize.Size `name:"db-sqlite-cache-size" usage:"Sqlite only: see https://www.sqlite.org/pragma.html#pragma_cache_size"`
|
||||
DbSqliteBusyTimeout time.Duration `name:"db-sqlite-busy-timeout" usage:"Sqlite only: see https://www.sqlite.org/pragma.html#pragma_busy_timeout"`
|
||||
|
||||
WebTemplateBaseDir string `name:"web-template-base-dir" usage:"Basedir for html templating files for rendering pages and composing emails."`
|
||||
WebAssetBaseDir string `name:"web-asset-base-dir" usage:"Directory to serve static assets from, accessible at example.org/assets/"`
|
||||
|
|
|
|||
|
|
@ -40,14 +40,18 @@ var Defaults = Configuration{
|
|||
Port: 8080,
|
||||
TrustedProxies: []string{"127.0.0.1/32", "::1"}, // localhost
|
||||
|
||||
DbType: "postgres",
|
||||
DbAddress: "",
|
||||
DbPort: 5432,
|
||||
DbUser: "",
|
||||
DbPassword: "",
|
||||
DbDatabase: "gotosocial",
|
||||
DbTLSMode: "disable",
|
||||
DbTLSCACert: "",
|
||||
DbType: "postgres",
|
||||
DbAddress: "",
|
||||
DbPort: 5432,
|
||||
DbUser: "",
|
||||
DbPassword: "",
|
||||
DbDatabase: "gotosocial",
|
||||
DbTLSMode: "disable",
|
||||
DbTLSCACert: "",
|
||||
DbSqliteJournalMode: "WAL",
|
||||
DbSqliteSynchronous: "NORMAL",
|
||||
DbSqliteCacheSize: 64 * bytesize.MiB,
|
||||
DbSqliteBusyTimeout: time.Second * 30,
|
||||
|
||||
WebTemplateBaseDir: "./web/template/",
|
||||
WebAssetBaseDir: "./web/assets/",
|
||||
|
|
|
|||
|
|
@ -51,6 +51,10 @@ func (s *ConfigState) AddGlobalFlags(cmd *cobra.Command) {
|
|||
cmd.PersistentFlags().String(DbDatabaseFlag(), cfg.DbDatabase, fieldtag("DbDatabase", "usage"))
|
||||
cmd.PersistentFlags().String(DbTLSModeFlag(), cfg.DbTLSMode, fieldtag("DbTLSMode", "usage"))
|
||||
cmd.PersistentFlags().String(DbTLSCACertFlag(), cfg.DbTLSCACert, fieldtag("DbTLSCACert", "usage"))
|
||||
cmd.PersistentFlags().String(DbSqliteJournalModeFlag(), cfg.DbSqliteJournalMode, fieldtag("DbSqliteJournalMode", "usage"))
|
||||
cmd.PersistentFlags().String(DbSqliteSynchronousFlag(), cfg.DbSqliteSynchronous, fieldtag("DbSqliteSynchronous", "usage"))
|
||||
cmd.PersistentFlags().Uint64(DbSqliteCacheSizeFlag(), uint64(cfg.DbSqliteCacheSize), fieldtag("DbSqliteCacheSize", "usage"))
|
||||
cmd.PersistentFlags().Duration(DbSqliteBusyTimeoutFlag(), cfg.DbSqliteBusyTimeout, fieldtag("DbSqliteBusyTimeout", "usage"))
|
||||
})
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -524,6 +524,106 @@ func GetDbTLSCACert() string { return global.GetDbTLSCACert() }
|
|||
// SetDbTLSCACert safely sets the value for global configuration 'DbTLSCACert' field
|
||||
func SetDbTLSCACert(v string) { global.SetDbTLSCACert(v) }
|
||||
|
||||
// GetDbSqliteJournalMode safely fetches the Configuration value for state's 'DbSqliteJournalMode' field
|
||||
func (st *ConfigState) GetDbSqliteJournalMode() (v string) {
|
||||
st.mutex.Lock()
|
||||
v = st.config.DbSqliteJournalMode
|
||||
st.mutex.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
// SetDbSqliteJournalMode safely sets the Configuration value for state's 'DbSqliteJournalMode' field
|
||||
func (st *ConfigState) SetDbSqliteJournalMode(v string) {
|
||||
st.mutex.Lock()
|
||||
defer st.mutex.Unlock()
|
||||
st.config.DbSqliteJournalMode = v
|
||||
st.reloadToViper()
|
||||
}
|
||||
|
||||
// DbSqliteJournalModeFlag returns the flag name for the 'DbSqliteJournalMode' field
|
||||
func DbSqliteJournalModeFlag() string { return "db-sqlite-journal-mode" }
|
||||
|
||||
// GetDbSqliteJournalMode safely fetches the value for global configuration 'DbSqliteJournalMode' field
|
||||
func GetDbSqliteJournalMode() string { return global.GetDbSqliteJournalMode() }
|
||||
|
||||
// SetDbSqliteJournalMode safely sets the value for global configuration 'DbSqliteJournalMode' field
|
||||
func SetDbSqliteJournalMode(v string) { global.SetDbSqliteJournalMode(v) }
|
||||
|
||||
// GetDbSqliteSynchronous safely fetches the Configuration value for state's 'DbSqliteSynchronous' field
|
||||
func (st *ConfigState) GetDbSqliteSynchronous() (v string) {
|
||||
st.mutex.Lock()
|
||||
v = st.config.DbSqliteSynchronous
|
||||
st.mutex.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
// SetDbSqliteSynchronous safely sets the Configuration value for state's 'DbSqliteSynchronous' field
|
||||
func (st *ConfigState) SetDbSqliteSynchronous(v string) {
|
||||
st.mutex.Lock()
|
||||
defer st.mutex.Unlock()
|
||||
st.config.DbSqliteSynchronous = v
|
||||
st.reloadToViper()
|
||||
}
|
||||
|
||||
// DbSqliteSynchronousFlag returns the flag name for the 'DbSqliteSynchronous' field
|
||||
func DbSqliteSynchronousFlag() string { return "db-sqlite-synchronous" }
|
||||
|
||||
// GetDbSqliteSynchronous safely fetches the value for global configuration 'DbSqliteSynchronous' field
|
||||
func GetDbSqliteSynchronous() string { return global.GetDbSqliteSynchronous() }
|
||||
|
||||
// SetDbSqliteSynchronous safely sets the value for global configuration 'DbSqliteSynchronous' field
|
||||
func SetDbSqliteSynchronous(v string) { global.SetDbSqliteSynchronous(v) }
|
||||
|
||||
// GetDbSqliteCacheSize safely fetches the Configuration value for state's 'DbSqliteCacheSize' field
|
||||
func (st *ConfigState) GetDbSqliteCacheSize() (v bytesize.Size) {
|
||||
st.mutex.Lock()
|
||||
v = st.config.DbSqliteCacheSize
|
||||
st.mutex.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
// SetDbSqliteCacheSize safely sets the Configuration value for state's 'DbSqliteCacheSize' field
|
||||
func (st *ConfigState) SetDbSqliteCacheSize(v bytesize.Size) {
|
||||
st.mutex.Lock()
|
||||
defer st.mutex.Unlock()
|
||||
st.config.DbSqliteCacheSize = v
|
||||
st.reloadToViper()
|
||||
}
|
||||
|
||||
// DbSqliteCacheSizeFlag returns the flag name for the 'DbSqliteCacheSize' field
|
||||
func DbSqliteCacheSizeFlag() string { return "db-sqlite-cache-size" }
|
||||
|
||||
// GetDbSqliteCacheSize safely fetches the value for global configuration 'DbSqliteCacheSize' field
|
||||
func GetDbSqliteCacheSize() bytesize.Size { return global.GetDbSqliteCacheSize() }
|
||||
|
||||
// SetDbSqliteCacheSize safely sets the value for global configuration 'DbSqliteCacheSize' field
|
||||
func SetDbSqliteCacheSize(v bytesize.Size) { global.SetDbSqliteCacheSize(v) }
|
||||
|
||||
// GetDbSqliteBusyTimeout safely fetches the Configuration value for state's 'DbSqliteBusyTimeout' field
|
||||
func (st *ConfigState) GetDbSqliteBusyTimeout() (v time.Duration) {
|
||||
st.mutex.Lock()
|
||||
v = st.config.DbSqliteBusyTimeout
|
||||
st.mutex.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
// SetDbSqliteBusyTimeout safely sets the Configuration value for state's 'DbSqliteBusyTimeout' field
|
||||
func (st *ConfigState) SetDbSqliteBusyTimeout(v time.Duration) {
|
||||
st.mutex.Lock()
|
||||
defer st.mutex.Unlock()
|
||||
st.config.DbSqliteBusyTimeout = v
|
||||
st.reloadToViper()
|
||||
}
|
||||
|
||||
// DbSqliteBusyTimeoutFlag returns the flag name for the 'DbSqliteBusyTimeout' field
|
||||
func DbSqliteBusyTimeoutFlag() string { return "db-sqlite-busy-timeout" }
|
||||
|
||||
// GetDbSqliteBusyTimeout safely fetches the value for global configuration 'DbSqliteBusyTimeout' field
|
||||
func GetDbSqliteBusyTimeout() time.Duration { return global.GetDbSqliteBusyTimeout() }
|
||||
|
||||
// SetDbSqliteBusyTimeout safely sets the value for global configuration 'DbSqliteBusyTimeout' field
|
||||
func SetDbSqliteBusyTimeout(v time.Duration) { global.SetDbSqliteBusyTimeout(v) }
|
||||
|
||||
// GetWebTemplateBaseDir safely fetches the Configuration value for state's 'WebTemplateBaseDir' field
|
||||
func (st *ConfigState) GetWebTemplateBaseDir() (v string) {
|
||||
st.mutex.Lock()
|
||||
|
|
|
|||
|
|
@ -28,9 +28,11 @@ import (
|
|||
"fmt"
|
||||
"os"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"codeberg.org/gruf/go-bytesize"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v4"
|
||||
"github.com/jackc/pgx/v4/stdlib"
|
||||
|
|
@ -49,22 +51,6 @@ import (
|
|||
"modernc.org/sqlite"
|
||||
)
|
||||
|
||||
const (
|
||||
dbTypePostgres = "postgres"
|
||||
dbTypeSqlite = "sqlite"
|
||||
|
||||
// dbTLSModeDisable does not attempt to make a TLS connection to the database.
|
||||
dbTLSModeDisable = "disable"
|
||||
// dbTLSModeEnable attempts to make a TLS connection to the database, but doesn't fail if
|
||||
// the certificate passed by the database isn't verified.
|
||||
dbTLSModeEnable = "enable"
|
||||
// dbTLSModeRequire attempts to make a TLS connection to the database, and requires
|
||||
// that the certificate presented by the database is valid.
|
||||
dbTLSModeRequire = "require"
|
||||
// dbTLSModeUnset means that the TLS mode has not been set.
|
||||
dbTLSModeUnset = ""
|
||||
)
|
||||
|
||||
var registerTables = []interface{}{
|
||||
>smodel.AccountToEmoji{},
|
||||
>smodel.StatusToEmoji{},
|
||||
|
|
@ -127,26 +113,34 @@ func doMigration(ctx context.Context, db *bun.DB) error {
|
|||
func NewBunDBService(ctx context.Context, state *state.State) (db.DB, error) {
|
||||
var conn *DBConn
|
||||
var err error
|
||||
dbType := strings.ToLower(config.GetDbType())
|
||||
t := strings.ToLower(config.GetDbType())
|
||||
|
||||
switch dbType {
|
||||
case dbTypePostgres:
|
||||
switch t {
|
||||
case "postgres":
|
||||
conn, err = pgConn(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
case dbTypeSqlite:
|
||||
case "sqlite":
|
||||
conn, err = sqliteConn(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
default:
|
||||
return nil, fmt.Errorf("database type %s not supported for bundb", dbType)
|
||||
return nil, fmt.Errorf("database type %s not supported for bundb", t)
|
||||
}
|
||||
|
||||
// Add database query hook
|
||||
conn.DB.AddQueryHook(queryHook{})
|
||||
|
||||
// execute sqlite pragmas *after* adding database hook;
|
||||
// this allows the pragma queries to be logged
|
||||
if t == "sqlite" {
|
||||
if err := sqlitePragmas(ctx, conn); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// table registration is needed for many-to-many, see:
|
||||
// https://bun.uptrace.dev/orm/many-to-many-relation/
|
||||
for _, t := range registerTables {
|
||||
|
|
@ -230,29 +224,29 @@ func NewBunDBService(ctx context.Context, state *state.State) (db.DB, error) {
|
|||
|
||||
func sqliteConn(ctx context.Context) (*DBConn, error) {
|
||||
// validate db address has actually been set
|
||||
dbAddress := config.GetDbAddress()
|
||||
if dbAddress == "" {
|
||||
address := config.GetDbAddress()
|
||||
if address == "" {
|
||||
return nil, fmt.Errorf("'%s' was not set when attempting to start sqlite", config.DbAddressFlag())
|
||||
}
|
||||
|
||||
// Drop anything fancy from DB address
|
||||
dbAddress = strings.Split(dbAddress, "?")[0]
|
||||
dbAddress = strings.TrimPrefix(dbAddress, "file:")
|
||||
address = strings.Split(address, "?")[0]
|
||||
address = strings.TrimPrefix(address, "file:")
|
||||
|
||||
// Append our own SQLite preferences
|
||||
dbAddress = "file:" + dbAddress + "?cache=shared"
|
||||
address = "file:" + address
|
||||
|
||||
var inMem bool
|
||||
|
||||
if dbAddress == "file::memory:?cache=shared" {
|
||||
dbAddress = fmt.Sprintf("file:%s?mode=memory&cache=shared", uuid.NewString())
|
||||
log.Infof("using in-memory database address " + dbAddress)
|
||||
if address == "file::memory:" {
|
||||
address = fmt.Sprintf("file:%s?mode=memory&cache=shared", uuid.NewString())
|
||||
log.Infof("using in-memory database address " + address)
|
||||
log.Warn("sqlite in-memory database should only be used for debugging")
|
||||
inMem = true
|
||||
}
|
||||
|
||||
// Open new DB instance
|
||||
sqldb, err := sql.Open("sqlite", dbAddress)
|
||||
sqldb, err := sql.Open("sqlite", address)
|
||||
if err != nil {
|
||||
if errWithCode, ok := err.(*sqlite.Error); ok {
|
||||
err = errors.New(sqlite.ErrorCodeString[errWithCode.Code()])
|
||||
|
|
@ -260,8 +254,6 @@ func sqliteConn(ctx context.Context) (*DBConn, error) {
|
|||
return nil, fmt.Errorf("could not open sqlite db: %s", err)
|
||||
}
|
||||
|
||||
tweakConnectionValues(sqldb)
|
||||
|
||||
if inMem {
|
||||
// don't close connections on disconnect -- otherwise
|
||||
// the SQLite database will be deleted when there
|
||||
|
|
@ -269,6 +261,7 @@ func sqliteConn(ctx context.Context) (*DBConn, error) {
|
|||
sqldb.SetConnMaxLifetime(0)
|
||||
}
|
||||
|
||||
// Wrap Bun database conn in our own wrapper
|
||||
conn := WrapDBConn(bun.NewDB(sqldb, sqlitedialect.New()))
|
||||
|
||||
// ping to check the db is there and listening
|
||||
|
|
@ -278,11 +271,56 @@ func sqliteConn(ctx context.Context) (*DBConn, error) {
|
|||
}
|
||||
return nil, fmt.Errorf("sqlite ping: %s", err)
|
||||
}
|
||||
|
||||
log.Info("connected to SQLITE database")
|
||||
|
||||
return conn, nil
|
||||
}
|
||||
|
||||
func sqlitePragmas(ctx context.Context, conn *DBConn) error {
|
||||
var pragmas [][]string
|
||||
if mode := config.GetDbSqliteJournalMode(); mode != "" {
|
||||
// Set the user provided SQLite journal mode
|
||||
pragmas = append(pragmas, []string{"journal_mode", mode})
|
||||
}
|
||||
|
||||
if mode := config.GetDbSqliteSynchronous(); mode != "" {
|
||||
// Set the user provided SQLite synchronous mode
|
||||
pragmas = append(pragmas, []string{"synchronous", mode})
|
||||
}
|
||||
|
||||
if size := config.GetDbSqliteCacheSize(); size > 0 {
|
||||
// Set the user provided SQLite cache size (in kibibytes)
|
||||
// Prepend a '-' character to this to indicate to sqlite
|
||||
// that we're giving kibibytes rather than num pages.
|
||||
// https://www.sqlite.org/pragma.html#pragma_cache_size
|
||||
s := "-" + strconv.FormatUint(uint64(size/bytesize.KiB), 10)
|
||||
pragmas = append(pragmas, []string{"cache_size", s})
|
||||
}
|
||||
|
||||
if timeout := config.GetDbSqliteBusyTimeout(); timeout > 0 {
|
||||
t := strconv.FormatInt(timeout.Milliseconds(), 10)
|
||||
pragmas = append(pragmas, []string{"busy_timeout", t})
|
||||
}
|
||||
|
||||
for _, p := range pragmas {
|
||||
pk := p[0]
|
||||
pv := p[1]
|
||||
|
||||
if _, err := conn.DB.ExecContext(ctx, "PRAGMA ?=?", bun.Ident(pk), bun.Safe(pv)); err != nil {
|
||||
return fmt.Errorf("error executing sqlite pragma %s: %w", pk, err)
|
||||
}
|
||||
|
||||
var res string
|
||||
if err := conn.DB.NewRaw("PRAGMA ?", bun.Ident(pk)).Scan(ctx, &res); err != nil {
|
||||
return fmt.Errorf("error scanning sqlite pragma %s: %w", pv, err)
|
||||
}
|
||||
|
||||
log.Infof("sqlite pragma %s set to %s", pk, res)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func pgConn(ctx context.Context) (*DBConn, error) {
|
||||
opts, err := deriveBunDBPGOptions() //nolint:contextcheck
|
||||
if err != nil {
|
||||
|
|
@ -291,7 +329,10 @@ func pgConn(ctx context.Context) (*DBConn, error) {
|
|||
|
||||
sqldb := stdlib.OpenDB(*opts)
|
||||
|
||||
tweakConnectionValues(sqldb)
|
||||
// https://bun.uptrace.dev/postgres/running-bun-in-production.html#database-sql
|
||||
maxOpenConns := 4 * runtime.GOMAXPROCS(0)
|
||||
sqldb.SetMaxOpenConns(maxOpenConns)
|
||||
sqldb.SetMaxIdleConns(maxOpenConns)
|
||||
|
||||
conn := WrapDBConn(bun.NewDB(sqldb, pgdialect.New()))
|
||||
|
||||
|
|
@ -311,10 +352,6 @@ func pgConn(ctx context.Context) (*DBConn, error) {
|
|||
// deriveBunDBPGOptions takes an application config and returns either a ready-to-use set of options
|
||||
// with sensible defaults, or an error if it's not satisfied by the provided config.
|
||||
func deriveBunDBPGOptions() (*pgx.ConnConfig, error) {
|
||||
if strings.ToUpper(config.GetDbType()) != db.DBTypePostgres {
|
||||
return nil, fmt.Errorf("expected db type of %s but got %s", db.DBTypePostgres, config.DbTypeFlag())
|
||||
}
|
||||
|
||||
// these are all optional, the db adapter figures out defaults
|
||||
address := config.GetDbAddress()
|
||||
|
||||
|
|
@ -326,14 +363,14 @@ func deriveBunDBPGOptions() (*pgx.ConnConfig, error) {
|
|||
|
||||
var tlsConfig *tls.Config
|
||||
switch config.GetDbTLSMode() {
|
||||
case dbTLSModeDisable, dbTLSModeUnset:
|
||||
case "", "disable":
|
||||
break // nothing to do
|
||||
case dbTLSModeEnable:
|
||||
case "enable":
|
||||
/* #nosec G402 */
|
||||
tlsConfig = &tls.Config{
|
||||
InsecureSkipVerify: true,
|
||||
}
|
||||
case dbTLSModeRequire:
|
||||
case "require":
|
||||
tlsConfig = &tls.Config{
|
||||
InsecureSkipVerify: false,
|
||||
ServerName: address,
|
||||
|
|
@ -397,13 +434,6 @@ func deriveBunDBPGOptions() (*pgx.ConnConfig, error) {
|
|||
return cfg, nil
|
||||
}
|
||||
|
||||
// https://bun.uptrace.dev/postgres/running-bun-in-production.html#database-sql
|
||||
func tweakConnectionValues(sqldb *sql.DB) {
|
||||
maxOpenConns := 4 * runtime.GOMAXPROCS(0)
|
||||
sqldb.SetMaxOpenConns(maxOpenConns)
|
||||
sqldb.SetMaxIdleConns(maxOpenConns)
|
||||
}
|
||||
|
||||
/*
|
||||
CONVERSION FUNCTIONS
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
set -eu
|
||||
|
||||
EXPECT='{"account-domain":"peepee","accounts-allow-custom-css":true,"accounts-approval-required":false,"accounts-reason-required":false,"accounts-registration-open":true,"advanced-cookies-samesite":"strict","advanced-rate-limit-requests":6969,"advanced-throttling-multiplier":-1,"application-name":"gts","bind-address":"127.0.0.1","cache":{"gts":{"account-max-size":99,"account-sweep-freq":1000000000,"account-ttl":10800000000000,"block-max-size":100,"block-sweep-freq":10000000000,"block-ttl":300000000000,"domain-block-max-size":1000,"domain-block-sweep-freq":60000000000,"domain-block-ttl":86400000000000,"emoji-category-max-size":100,"emoji-category-sweep-freq":10000000000,"emoji-category-ttl":300000000000,"emoji-max-size":500,"emoji-sweep-freq":10000000000,"emoji-ttl":300000000000,"mention-max-size":500,"mention-sweep-freq":10000000000,"mention-ttl":300000000000,"notification-max-size":500,"notification-sweep-freq":10000000000,"notification-ttl":300000000000,"report-max-size":100,"report-sweep-freq":10000000000,"report-ttl":300000000000,"status-max-size":500,"status-sweep-freq":10000000000,"status-ttl":300000000000,"tombstone-max-size":100,"tombstone-sweep-freq":10000000000,"tombstone-ttl":300000000000,"user-max-size":100,"user-sweep-freq":10000000000,"user-ttl":300000000000}},"config-path":"internal/config/testdata/test.yaml","db-address":":memory:","db-database":"gotosocial_prod","db-password":"hunter2","db-port":6969,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"sqlite","db-user":"sex-haver","dry-run":false,"email":"","host":"example.com","instance-deliver-to-shared-inboxes":false,"instance-expose-peers":true,"instance-expose-public-timeline":true,"instance-expose-suspended":true,"landing-page-user":"admin","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":true,"log-level":"info","media-description-max-chars":5000,"media-description-min-chars":69,"media-emoji-local-max-size":420,"media-emoji-remote-max-size":420,"media-image-max-size":420,"media-remote-cache-days":30,"media-video-max-size":420,"oidc-client-id":"1234","oidc-client-secret":"shhhh its a secret","oidc-enabled":true,"oidc-idp-name":"sex-haver","oidc-issuer":"whoknows","oidc-link-existing":true,"oidc-scopes":["read","write"],"oidc-skip-verification":true,"password":"","path":"","port":6969,"protocol":"http","smtp-from":"queen.rip.in.piss@terfisland.org","smtp-host":"example.com","smtp-password":"hunter2","smtp-port":4269,"smtp-username":"sex-haver","software-version":"","statuses-cw-max-chars":420,"statuses-max-chars":69,"statuses-media-max-files":1,"statuses-poll-max-options":1,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/root/store","storage-s3-access-key":"minio","storage-s3-bucket":"gts","storage-s3-endpoint":"localhost:9000","storage-s3-proxy":true,"storage-s3-secret-key":"miniostorage","storage-s3-use-ssl":false,"syslog-address":"127.0.0.1:6969","syslog-enabled":true,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","docker.host.local"],"username":"","web-asset-base-dir":"/root","web-template-base-dir":"/root"}'
|
||||
EXPECT='{"account-domain":"peepee","accounts-allow-custom-css":true,"accounts-approval-required":false,"accounts-reason-required":false,"accounts-registration-open":true,"advanced-cookies-samesite":"strict","advanced-rate-limit-requests":6969,"advanced-throttling-multiplier":-1,"application-name":"gts","bind-address":"127.0.0.1","cache":{"gts":{"account-max-size":99,"account-sweep-freq":1000000000,"account-ttl":10800000000000,"block-max-size":100,"block-sweep-freq":10000000000,"block-ttl":300000000000,"domain-block-max-size":1000,"domain-block-sweep-freq":60000000000,"domain-block-ttl":86400000000000,"emoji-category-max-size":100,"emoji-category-sweep-freq":10000000000,"emoji-category-ttl":300000000000,"emoji-max-size":500,"emoji-sweep-freq":10000000000,"emoji-ttl":300000000000,"mention-max-size":500,"mention-sweep-freq":10000000000,"mention-ttl":300000000000,"notification-max-size":500,"notification-sweep-freq":10000000000,"notification-ttl":300000000000,"report-max-size":100,"report-sweep-freq":10000000000,"report-ttl":300000000000,"status-max-size":500,"status-sweep-freq":10000000000,"status-ttl":300000000000,"tombstone-max-size":100,"tombstone-sweep-freq":10000000000,"tombstone-ttl":300000000000,"user-max-size":100,"user-sweep-freq":10000000000,"user-ttl":300000000000}},"config-path":"internal/config/testdata/test.yaml","db-address":":memory:","db-database":"gotosocial_prod","db-password":"hunter2","db-port":6969,"db-sqlite-busy-timeout":1000000000,"db-sqlite-cache-size":0,"db-sqlite-journal-mode":"DELETE","db-sqlite-synchronous":"FULL","db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"sqlite","db-user":"sex-haver","dry-run":false,"email":"","host":"example.com","instance-deliver-to-shared-inboxes":false,"instance-expose-peers":true,"instance-expose-public-timeline":true,"instance-expose-suspended":true,"landing-page-user":"admin","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":true,"log-level":"info","media-description-max-chars":5000,"media-description-min-chars":69,"media-emoji-local-max-size":420,"media-emoji-remote-max-size":420,"media-image-max-size":420,"media-remote-cache-days":30,"media-video-max-size":420,"oidc-client-id":"1234","oidc-client-secret":"shhhh its a secret","oidc-enabled":true,"oidc-idp-name":"sex-haver","oidc-issuer":"whoknows","oidc-link-existing":true,"oidc-scopes":["read","write"],"oidc-skip-verification":true,"password":"","path":"","port":6969,"protocol":"http","smtp-from":"queen.rip.in.piss@terfisland.org","smtp-host":"example.com","smtp-password":"hunter2","smtp-port":4269,"smtp-username":"sex-haver","software-version":"","statuses-cw-max-chars":420,"statuses-max-chars":69,"statuses-media-max-files":1,"statuses-poll-max-options":1,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/root/store","storage-s3-access-key":"minio","storage-s3-bucket":"gts","storage-s3-endpoint":"localhost:9000","storage-s3-proxy":true,"storage-s3-secret-key":"miniostorage","storage-s3-use-ssl":false,"syslog-address":"127.0.0.1:6969","syslog-enabled":true,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","docker.host.local"],"username":"","web-asset-base-dir":"/root","web-template-base-dir":"/root"}'
|
||||
|
||||
# Set all the environment variables to
|
||||
# ensure that these are parsed without panic
|
||||
|
|
@ -22,6 +22,10 @@ GTS_DB_PORT=6969 \
|
|||
GTS_DB_USER='sex-haver' \
|
||||
GTS_DB_PASSWORD='hunter2' \
|
||||
GTS_DB_DATABASE='gotosocial_prod' \
|
||||
GTS_DB_SQLITE_JOURNAL_MODE='DELETE' \
|
||||
GTS_DB_SQLITE_SYNCHRONOUS='FULL' \
|
||||
GTS_DB_SQLITE_CACHE_SIZE=0 \
|
||||
GTS_DB_SQLITE_BUSY_TIMEOUT='1s' \
|
||||
GTS_TLS_MODE='' \
|
||||
GTS_DB_TLS_CA_CERT='' \
|
||||
GTS_WEB_TEMPLATE_BASE_DIR='/root' \
|
||||
|
|
|
|||
|
|
@ -19,6 +19,9 @@
|
|||
package testrig
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"codeberg.org/gruf/go-bytesize"
|
||||
"github.com/coreos/go-oidc/v3/oidc"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/config"
|
||||
)
|
||||
|
|
@ -43,12 +46,16 @@ var testDefaults = config.Configuration{
|
|||
Port: 8080,
|
||||
TrustedProxies: []string{"127.0.0.1/32", "::1"},
|
||||
|
||||
DbType: "sqlite",
|
||||
DbAddress: ":memory:",
|
||||
DbPort: 5432,
|
||||
DbUser: "postgres",
|
||||
DbPassword: "postgres",
|
||||
DbDatabase: "postgres",
|
||||
DbType: "sqlite",
|
||||
DbAddress: ":memory:",
|
||||
DbPort: 5432,
|
||||
DbUser: "postgres",
|
||||
DbPassword: "postgres",
|
||||
DbDatabase: "postgres",
|
||||
DbSqliteJournalMode: "WAL",
|
||||
DbSqliteSynchronous: "NORMAL",
|
||||
DbSqliteCacheSize: 64 * bytesize.MiB,
|
||||
DbSqliteBusyTimeout: time.Second * 30,
|
||||
|
||||
WebTemplateBaseDir: "./web/template/",
|
||||
WebAssetBaseDir: "./web/assets/",
|
||||
|
|
|
|||
|
|
@ -73,15 +73,11 @@ var testModels = []interface{}{
|
|||
// value as the port instead.
|
||||
func NewTestDB() db.DB {
|
||||
if alternateAddress := os.Getenv("GTS_DB_ADDRESS"); alternateAddress != "" {
|
||||
config.Config(func(cfg *config.Configuration) {
|
||||
cfg.DbAddress = alternateAddress
|
||||
})
|
||||
config.SetDbAddress(alternateAddress)
|
||||
}
|
||||
|
||||
if alternateDBType := os.Getenv("GTS_DB_TYPE"); alternateDBType != "" {
|
||||
config.Config(func(cfg *config.Configuration) {
|
||||
cfg.DbType = alternateDBType
|
||||
})
|
||||
config.SetDbType(alternateDBType)
|
||||
}
|
||||
|
||||
if alternateDBPort := os.Getenv("GTS_DB_PORT"); alternateDBPort != "" {
|
||||
|
|
@ -89,9 +85,7 @@ func NewTestDB() db.DB {
|
|||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
config.Config(func(cfg *config.Configuration) {
|
||||
cfg.DbPort = int(port)
|
||||
})
|
||||
config.SetDbPort(int(port))
|
||||
}
|
||||
|
||||
var state state.State
|
||||
|
|
|
|||
Loading…
Reference in New Issue