update go-store to latest

This commit is contained in:
tsmethurst 2022-01-29 12:15:51 +01:00
parent 9aa364f1eb
commit 4e74c84148
8 changed files with 70 additions and 52 deletions

2
go.mod
View File

@ -4,7 +4,7 @@ go 1.17
require ( require (
codeberg.org/gruf/go-runners v1.2.0 codeberg.org/gruf/go-runners v1.2.0
codeberg.org/gruf/go-store v1.3.2 codeberg.org/gruf/go-store v1.3.3
github.com/ReneKroon/ttlcache v1.7.0 github.com/ReneKroon/ttlcache v1.7.0
github.com/buckket/go-blurhash v1.1.0 github.com/buckket/go-blurhash v1.1.0
github.com/coreos/go-oidc/v3 v3.1.0 github.com/coreos/go-oidc/v3 v3.1.0

2
go.sum
View File

@ -71,6 +71,8 @@ codeberg.org/gruf/go-runners v1.2.0 h1:tkoPrwYMkVg1o/C4PGTR1YbC11XX4r06uLPOYajBs
codeberg.org/gruf/go-runners v1.2.0/go.mod h1:9gTrmMnO3d+50C+hVzcmGBf+zTuswReS278E2EMvnmw= codeberg.org/gruf/go-runners v1.2.0/go.mod h1:9gTrmMnO3d+50C+hVzcmGBf+zTuswReS278E2EMvnmw=
codeberg.org/gruf/go-store v1.3.2 h1:cLTMEqyK0uF/bt1ULkRR4h41Pdgxwvw3uxSpLUublHo= codeberg.org/gruf/go-store v1.3.2 h1:cLTMEqyK0uF/bt1ULkRR4h41Pdgxwvw3uxSpLUublHo=
codeberg.org/gruf/go-store v1.3.2/go.mod h1:g4+9h3wbwZ6IW0uhpw57xywcqiy4CIj0zQLqqtjEU1M= codeberg.org/gruf/go-store v1.3.2/go.mod h1:g4+9h3wbwZ6IW0uhpw57xywcqiy4CIj0zQLqqtjEU1M=
codeberg.org/gruf/go-store v1.3.3 h1:fAP9FXy6HiLPxdD7cmpSzyfKXmVvZLjqn0m7HhxVT5M=
codeberg.org/gruf/go-store v1.3.3/go.mod h1:g4+9h3wbwZ6IW0uhpw57xywcqiy4CIj0zQLqqtjEU1M=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=

View File

@ -140,7 +140,7 @@ func OpenBlock(path string, cfg *BlockConfig) (*BlockStorage, error) {
} }
// Open and acquire storage lock for path // Open and acquire storage lock for path
lock, err := OpenLock(pb.Join(path, lockFile)) lock, err := OpenLock(pb.Join(path, LockFile))
if err != nil { if err != nil {
return nil, err return nil, err
} }

View File

@ -5,6 +5,8 @@ import (
"io/fs" "io/fs"
"os" "os"
"path" "path"
_path "path"
"strings"
"syscall" "syscall"
"codeberg.org/gruf/go-bytes" "codeberg.org/gruf/go-bytes"
@ -31,6 +33,11 @@ type DiskConfig struct {
// Overwrite allows overwriting values of stored keys in the storage // Overwrite allows overwriting values of stored keys in the storage
Overwrite bool Overwrite bool
// LockFile allows specifying the filesystem path to use for the lockfile,
// providing only a filename it will store the lockfile within provided store
// path and nest the store under `path/store` to prevent access to lockfile
LockFile string
// Compression is the Compressor to use when reading / writing files, default is no compression // Compression is the Compressor to use when reading / writing files, default is no compression
Compression Compressor Compression Compressor
} }
@ -57,11 +64,17 @@ func getDiskConfig(cfg *DiskConfig) DiskConfig {
cfg.WriteBufSize = DefaultDiskConfig.WriteBufSize cfg.WriteBufSize = DefaultDiskConfig.WriteBufSize
} }
// Assume empty lockfile path == use default
if len(cfg.LockFile) < 1 {
cfg.LockFile = LockFile
}
// Return owned config copy // Return owned config copy
return DiskConfig{ return DiskConfig{
Transform: cfg.Transform, Transform: cfg.Transform,
WriteBufSize: cfg.WriteBufSize, WriteBufSize: cfg.WriteBufSize,
Overwrite: cfg.Overwrite, Overwrite: cfg.Overwrite,
LockFile: cfg.LockFile,
Compression: cfg.Compression, Compression: cfg.Compression,
} }
} }
@ -76,16 +89,27 @@ type DiskStorage struct {
// OpenFile opens a DiskStorage instance for given folder path and configuration // OpenFile opens a DiskStorage instance for given folder path and configuration
func OpenFile(path string, cfg *DiskConfig) (*DiskStorage, error) { func OpenFile(path string, cfg *DiskConfig) (*DiskStorage, error) {
// Get checked config
config := getDiskConfig(cfg)
// Acquire path builder // Acquire path builder
pb := util.GetPathBuilder() pb := util.GetPathBuilder()
defer util.PutPathBuilder(pb) defer util.PutPathBuilder(pb)
// Clean provided path, ensure ends in '/' (should // Clean provided store path, ensure
// be dir, this helps with file path trimming later) // ends in '/' to help later path trimming
storePath := pb.Join(path, "store") + "/" storePath := pb.Clean(path) + "/"
// Get checked config // Clean provided lockfile path
config := getDiskConfig(cfg) lockfile := pb.Clean(config.LockFile)
// Check if lockfile is an *actual* path or just filename
if lockDir, _ := _path.Split(lockfile); len(lockDir) < 1 {
// Lockfile is a filename, store must be nested under
// $storePath/store to prevent access to the lockfile
storePath += "store/"
lockfile = pb.Join(path, lockfile)
}
// Attempt to open dir path // Attempt to open dir path
file, err := os.OpenFile(storePath, defaultFileROFlags, defaultDirPerms) file, err := os.OpenFile(storePath, defaultFileROFlags, defaultDirPerms)
@ -118,7 +142,7 @@ func OpenFile(path string, cfg *DiskConfig) (*DiskStorage, error) {
} }
// Open and acquire storage lock for path // Open and acquire storage lock for path
lock, err := OpenLock(pb.Join(path, lockFile)) lock, err := OpenLock(lockfile)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -347,9 +371,27 @@ func (st *DiskStorage) filepath(key string) (string, error) {
pb.AppendString(key) pb.AppendString(key)
// Check for dir traversal outside of root // Check for dir traversal outside of root
if util.IsDirTraversal(st.path, pb.StringPtr()) { if isDirTraversal(st.path, pb.StringPtr()) {
return "", ErrInvalidKey return "", ErrInvalidKey
} }
return pb.String(), nil return pb.String(), nil
} }
// isDirTraversal will check if rootPlusPath is a dir traversal outside of root,
// assuming that both are cleaned and that rootPlusPath is path.Join(root, somePath)
func isDirTraversal(root, rootPlusPath string) bool {
switch {
// Root is $PWD, check for traversal out of
case root == ".":
return strings.HasPrefix(rootPlusPath, "../")
// The path MUST be prefixed by root
case !strings.HasPrefix(rootPlusPath, root):
return true
// In all other cases, check not equal
default:
return len(root) == len(rootPlusPath)
}
}

View File

@ -8,13 +8,8 @@ import (
"codeberg.org/gruf/go-store/util" "codeberg.org/gruf/go-store/util"
) )
// lockFile is our standard lockfile name. // LockFile is our standard lockfile name.
var lockFile = "store.lock" const LockFile = "store.lock"
// IsLockKey returns whether storage key is our lockfile.
func IsLockKey(key string) bool {
return key == lockFile
}
// Lock represents a filesystem lock to ensure only one storage instance open per path. // Lock represents a filesystem lock to ensure only one storage instance open per path.
type Lock struct { type Lock struct {

View File

@ -3,30 +3,10 @@ package util
import ( import (
"io/fs" "io/fs"
"os" "os"
"strings"
"syscall"
"codeberg.org/gruf/go-fastpath" "codeberg.org/gruf/go-fastpath"
) )
// IsDirTraversal will check if rootPlusPath is a dir traversal outside of root,
// assuming that both are cleaned and that rootPlusPath is path.Join(root, somePath)
func IsDirTraversal(root string, rootPlusPath string) bool {
switch {
// Root is $PWD, check for traversal out of
case root == ".":
return strings.HasPrefix(rootPlusPath, "../")
// The path MUST be prefixed by root
case !strings.HasPrefix(rootPlusPath, root):
return true
// In all other cases, check not equal
default:
return len(root) == len(rootPlusPath)
}
}
// WalkDir traverses the dir tree of the supplied path, performing the supplied walkFn on each entry // WalkDir traverses the dir tree of the supplied path, performing the supplied walkFn on each entry
func WalkDir(pb *fastpath.Builder, path string, walkFn func(string, fs.DirEntry)) error { func WalkDir(pb *fastpath.Builder, path string, walkFn func(string, fs.DirEntry)) error {
// Read supplied dir path // Read supplied dir path
@ -100,14 +80,3 @@ func cleanDirs(pb *fastpath.Builder, path string) error {
} }
return nil return nil
} }
// RetryOnEINTR is a low-level filesystem function for retrying syscalls on O_EINTR received
func RetryOnEINTR(do func() error) error {
for {
err := do()
if err == syscall.EINTR {
continue
}
return err
}
}

14
vendor/codeberg.org/gruf/go-store/util/sys.go generated vendored Normal file
View File

@ -0,0 +1,14 @@
package util
import "syscall"
// RetryOnEINTR is a low-level filesystem function for retrying syscalls on O_EINTR received
func RetryOnEINTR(do func() error) error {
for {
err := do()
if err == syscall.EINTR {
continue
}
return err
}
}

6
vendor/modules.txt vendored
View File

@ -16,15 +16,13 @@ codeberg.org/gruf/go-hashenc
# codeberg.org/gruf/go-mutexes v1.1.0 # codeberg.org/gruf/go-mutexes v1.1.0
## explicit; go 1.14 ## explicit; go 1.14
codeberg.org/gruf/go-mutexes codeberg.org/gruf/go-mutexes
# codeberg.org/gruf/go-nowish v1.1.0
## explicit; go 1.14
# codeberg.org/gruf/go-pools v1.0.2 # codeberg.org/gruf/go-pools v1.0.2
## explicit; go 1.16 ## explicit; go 1.16
codeberg.org/gruf/go-pools codeberg.org/gruf/go-pools
# codeberg.org/gruf/go-runners v1.2.0 # codeberg.org/gruf/go-runners v1.2.0
## explicit; go 1.14 ## explicit; go 1.14
codeberg.org/gruf/go-runners codeberg.org/gruf/go-runners
# codeberg.org/gruf/go-store v1.3.2 # codeberg.org/gruf/go-store v1.3.3
## explicit; go 1.14 ## explicit; go 1.14
codeberg.org/gruf/go-store/kv codeberg.org/gruf/go-store/kv
codeberg.org/gruf/go-store/storage codeberg.org/gruf/go-store/storage
@ -521,8 +519,6 @@ github.com/vmihailenco/tagparser/v2/internal/parser
# github.com/wagslane/go-password-validator v0.3.0 # github.com/wagslane/go-password-validator v0.3.0
## explicit; go 1.16 ## explicit; go 1.16
github.com/wagslane/go-password-validator github.com/wagslane/go-password-validator
# github.com/zeebo/blake3 v0.2.1
## explicit; go 1.13
# golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b # golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b
## explicit; go 1.17 ## explicit; go 1.17
golang.org/x/crypto/acme golang.org/x/crypto/acme