update go-sqlite3 => v0.20.0 (#3483)

This commit is contained in:
kim
2024-10-25 16:09:18 +00:00
committed by GitHub
parent d8a83860bc
commit 51cb6cae16
41 changed files with 841 additions and 263 deletions

View File

@ -4,7 +4,7 @@ This package implements the SQLite [OS Interface](https://sqlite.org/vfs.html) (
It replaces the default SQLite VFS with a **pure Go** implementation,
and exposes [interfaces](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs#VFS)
that should allow you to implement your own custom VFSes.
that should allow you to implement your own [custom VFSes](#custom-vfses).
Since it is a from scratch reimplementation,
there are naturally some ways it deviates from the original.
@ -16,12 +16,12 @@ The main differences are [file locking](#file-locking) and [WAL mode](#write-ahe
POSIX advisory locks, which SQLite uses on Unix, are
[broken by design](https://github.com/sqlite/sqlite/blob/b74eb0/src/os_unix.c#L1073-L1161).
On Linux and macOS, this module uses
On Linux and macOS, this package uses
[OFD locks](https://www.gnu.org/software/libc/manual/html_node/Open-File-Description-Locks.html)
to synchronize access to database files.
OFD locks are fully compatible with POSIX advisory locks.
This module can also use
This package can also use
[BSD locks](https://man.freebsd.org/cgi/man.cgi?query=flock&sektion=2),
albeit with reduced concurrency (`BEGIN IMMEDIATE` behaves like `BEGIN EXCLUSIVE`).
On BSD, macOS, and illumos, BSD locks are fully compatible with POSIX advisory locks;
@ -30,7 +30,7 @@ elsewhere, they are very likely broken.
BSD locks are the default on BSD and illumos,
but you can opt into them with the `sqlite3_flock` build tag.
On Windows, this module uses `LockFileEx` and `UnlockFileEx`,
On Windows, this package uses `LockFileEx` and `UnlockFileEx`,
like SQLite.
Otherwise, file locking is not supported, and you must use
@ -46,18 +46,14 @@ to check if your build supports file locking.
### Write-Ahead Logging
On 64-bit little-endian Unix, this module uses `mmap` to implement
On little-endian Unix, this package uses `mmap` to implement
[shared-memory for the WAL-index](https://sqlite.org/wal.html#implementation_of_shared_memory_for_the_wal_index),
like SQLite.
To allow `mmap` to work, each connection needs to reserve up to 4GB of address space.
To limit the address space each connection reserves,
use [`WithMemoryLimitPages`](../tests/testcfg/testcfg.go).
With [BSD locks](https://man.freebsd.org/cgi/man.cgi?query=flock&sektion=2)
a WAL database can only be accessed by a single proccess.
Other processes that attempt to access a database locked with BSD locks,
will fail with the `SQLITE_PROTOCOL` error code.
will fail with the [`SQLITE_PROTOCOL`](https://sqlite.org/rescode.html#protocol) error code.
Otherwise, [WAL support is limited](https://sqlite.org/wal.html#noshm),
and `EXCLUSIVE` locking mode must be set to create, read, and write WAL databases.
@ -71,9 +67,22 @@ to check if your build supports shared memory.
### Batch-Atomic Write
On 64-bit Linux, this module supports [batch-atomic writes](https://sqlite.org/cgi/src/technote/714)
On 64-bit Linux, this package supports
[batch-atomic writes](https://sqlite.org/cgi/src/technote/714)
on the F2FS filesystem.
### Checksums
This package can be [configured](https://pkg.go.dev/github.com/ncruces/go-sqlite3#Conn.EnableChecksums)
to add an 8-byte checksum to the end of every page in an SQLite database.
The checksum is added as each page is written
and verified as each page is read.\
The checksum is intended to help detect database corruption
caused by random bit-flips in the mass storage device.
The implementation is compatible with SQLite's
[Checksum VFS Shim](https://sqlite.org/cksumvfs.html).
### Build Tags
The VFS can be customized with a few build tags:
@ -90,3 +99,14 @@ The VFS can be customized with a few build tags:
> [`unix-flock` VFS](https://sqlite.org/compile.html#enable_locking_style).
> If incompatible file locking is used, accessing databases concurrently with
> _other_ SQLite libraries will eventually corrupt data.
### Custom VFSes
- [`github.com/ncruces/go-sqlite3/vfs/adiantum`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/adiantum)
wraps a VFS to offer encryption at rest.
- [`github.com/ncruces/go-sqlite3/vfs/memdb`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/memdb)
implements an in-memory VFS.
- [`github.com/ncruces/go-sqlite3/vfs/readervfs`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/readervfs)
implements a VFS for immutable databases.
- [`github.com/ncruces/go-sqlite3/vfs/xts`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/xts)
wraps a VFS to offer encryption at rest.

View File

@ -49,6 +49,13 @@ type File interface {
DeviceCharacteristics() DeviceCharacteristic
}
// FileUnwrap should be implemented by a File
// that wraps another File implementation.
type FileUnwrap interface {
File
Unwrap() File
}
// FileLockState extends File to implement the
// SQLITE_FCNTL_LOCKSTATE file control opcode.
//
@ -58,6 +65,26 @@ type FileLockState interface {
LockState() LockLevel
}
// FilePersistentWAL extends File to implement the
// SQLITE_FCNTL_PERSIST_WAL file control opcode.
//
// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntlpersistwal
type FilePersistentWAL interface {
File
PersistentWAL() bool
SetPersistentWAL(bool)
}
// FilePowersafeOverwrite extends File to implement the
// SQLITE_FCNTL_POWERSAFE_OVERWRITE file control opcode.
//
// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntlpowersafeoverwrite
type FilePowersafeOverwrite interface {
File
PowersafeOverwrite() bool
SetPowersafeOverwrite(bool)
}
// FileChunkSize extends File to implement the
// SQLITE_FCNTL_CHUNK_SIZE file control opcode.
//
@ -94,26 +121,6 @@ type FileOverwrite interface {
Overwrite() error
}
// FilePersistentWAL extends File to implement the
// SQLITE_FCNTL_PERSIST_WAL file control opcode.
//
// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntlpersistwal
type FilePersistentWAL interface {
File
PersistentWAL() bool
SetPersistentWAL(bool)
}
// FilePowersafeOverwrite extends File to implement the
// SQLITE_FCNTL_POWERSAFE_OVERWRITE file control opcode.
//
// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntlpowersafeoverwrite
type FilePowersafeOverwrite interface {
File
PowersafeOverwrite() bool
SetPowersafeOverwrite(bool)
}
// FileCommitPhaseTwo extends File to implement the
// SQLITE_FCNTL_COMMIT_PHASETWO file control opcode.
//
@ -135,15 +142,6 @@ type FileBatchAtomicWrite interface {
RollbackAtomicWrite() error
}
// FilePragma extends File to implement the
// SQLITE_FCNTL_PRAGMA file control opcode.
//
// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntlpragma
type FilePragma interface {
File
Pragma(name, value string) (string, error)
}
// FileCheckpoint extends File to implement the
// SQLITE_FCNTL_CKPT_START and SQLITE_FCNTL_CKPT_DONE
// file control opcodes.
@ -151,8 +149,17 @@ type FilePragma interface {
// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntlckptstart
type FileCheckpoint interface {
File
CheckpointDone() error
CheckpointStart() error
CheckpointStart()
CheckpointDone()
}
// FilePragma extends File to implement the
// SQLITE_FCNTL_PRAGMA file control opcode.
//
// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntlpragma
type FilePragma interface {
File
Pragma(name, value string) (string, error)
}
// FileSharedMemory extends File to possibly implement
@ -171,5 +178,16 @@ type SharedMemory interface {
shmMap(context.Context, api.Module, int32, int32, bool) (uint32, _ErrorCode)
shmLock(int32, int32, _ShmFlag) _ErrorCode
shmUnmap(bool)
shmBarrier()
io.Closer
}
type blockingSharedMemory interface {
SharedMemory
shmEnableBlocking(block bool)
}
type fileControl interface {
File
fileControl(ctx context.Context, mod api.Module, op _FcntlOpcode, pArg uint32) _ErrorCode
}

149
vendor/github.com/ncruces/go-sqlite3/vfs/cksm.go generated vendored Normal file
View File

@ -0,0 +1,149 @@
package vfs
import (
"bytes"
"context"
_ "embed"
"encoding/binary"
"strconv"
"github.com/tetratelabs/wazero/api"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/ncruces/go-sqlite3/util/sql3util"
)
func cksmWrapFile(name *Filename, flags OpenFlag, file File) File {
// Checksum only main databases and WALs.
if flags&(OPEN_MAIN_DB|OPEN_WAL) == 0 {
return file
}
cksm := cksmFile{File: file}
if flags&OPEN_WAL != 0 {
main, _ := name.DatabaseFile().(cksmFile)
cksm.cksmFlags = main.cksmFlags
} else {
cksm.cksmFlags = new(cksmFlags)
cksm.isDB = true
}
return cksm
}
type cksmFile struct {
File
*cksmFlags
isDB bool
}
type cksmFlags struct {
computeCksm bool
verifyCksm bool
inCkpt bool
pageSize int
}
func (c cksmFile) ReadAt(p []byte, off int64) (n int, err error) {
n, err = c.File.ReadAt(p, off)
// SQLite is reading the header of a database file.
if c.isDB && off == 0 && len(p) >= 100 &&
bytes.HasPrefix(p, []byte("SQLite format 3\000")) {
c.init(p)
}
// Verify checksums.
if c.verifyCksm && !c.inCkpt && len(p) == c.pageSize {
cksm1 := cksmCompute(p[:len(p)-8])
cksm2 := *(*[8]byte)(p[len(p)-8:])
if cksm1 != cksm2 {
return 0, _IOERR_DATA
}
}
return n, err
}
func (c cksmFile) WriteAt(p []byte, off int64) (n int, err error) {
// SQLite is writing the first page of a database file.
if c.isDB && off == 0 && len(p) >= 100 &&
bytes.HasPrefix(p, []byte("SQLite format 3\000")) {
c.init(p)
}
// Compute checksums.
if c.computeCksm && !c.inCkpt && len(p) == c.pageSize {
*(*[8]byte)(p[len(p)-8:]) = cksmCompute(p[:len(p)-8])
}
return c.File.WriteAt(p, off)
}
func (c cksmFile) Pragma(name string, value string) (string, error) {
switch name {
case "checksum_verification":
b, ok := sql3util.ParseBool(value)
if ok {
c.verifyCksm = b && c.computeCksm
}
if !c.verifyCksm {
return "0", nil
}
return "1", nil
case "page_size":
if c.computeCksm {
// Do not allow page size changes on a checksum database.
return strconv.Itoa(c.pageSize), nil
}
}
return "", _NOTFOUND
}
func (c cksmFile) fileControl(ctx context.Context, mod api.Module, op _FcntlOpcode, pArg uint32) _ErrorCode {
switch op {
case _FCNTL_CKPT_START:
c.inCkpt = true
case _FCNTL_CKPT_DONE:
c.inCkpt = false
}
if rc := vfsFileControlImpl(ctx, mod, c, op, pArg); rc != _NOTFOUND {
return rc
}
return vfsFileControlImpl(ctx, mod, c.File, op, pArg)
}
func (f *cksmFlags) init(header []byte) {
f.pageSize = 256 * int(binary.LittleEndian.Uint16(header[16:18]))
if r := header[20] == 8; r != f.computeCksm {
f.computeCksm = r
f.verifyCksm = r
}
}
func cksmCompute(a []byte) (cksm [8]byte) {
var s1, s2 uint32
for len(a) >= 8 {
s1 += binary.LittleEndian.Uint32(a[0:4]) + s2
s2 += binary.LittleEndian.Uint32(a[4:8]) + s1
a = a[8:]
}
if len(a) != 0 {
panic(util.AssertErr())
}
binary.LittleEndian.PutUint32(cksm[0:4], s1)
binary.LittleEndian.PutUint32(cksm[4:8], s2)
return
}
func (c cksmFile) SharedMemory() SharedMemory {
if f, ok := c.File.(FileSharedMemory); ok {
return f.SharedMemory()
}
return nil
}
func (c cksmFile) Unwrap() File {
return c.File
}

View File

@ -51,6 +51,7 @@ const (
_IOERR_BEGIN_ATOMIC _ErrorCode = util.IOERR_BEGIN_ATOMIC
_IOERR_COMMIT_ATOMIC _ErrorCode = util.IOERR_COMMIT_ATOMIC
_IOERR_ROLLBACK_ATOMIC _ErrorCode = util.IOERR_ROLLBACK_ATOMIC
_IOERR_DATA _ErrorCode = util.IOERR_DATA
_BUSY_SNAPSHOT _ErrorCode = util.BUSY_SNAPSHOT
_CANTOPEN_FULLPATH _ErrorCode = util.CANTOPEN_FULLPATH
_CANTOPEN_ISDIR _ErrorCode = util.CANTOPEN_ISDIR

View File

@ -4,8 +4,9 @@ import (
"context"
"net/url"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/tetratelabs/wazero/api"
"github.com/ncruces/go-sqlite3/internal/util"
)
// Filename is used by SQLite to pass filenames

View File

@ -1,4 +1,4 @@
//go:build (amd64 || arm64 || riscv64 || ppc64le) && !sqlite3_nosys
//go:build (amd64 || arm64 || riscv64) && !sqlite3_nosys
package vfs
@ -13,7 +13,7 @@ const (
_F2FS_IOC_START_ATOMIC_WRITE = 62721
_F2FS_IOC_COMMIT_ATOMIC_WRITE = 62722
_F2FS_IOC_ABORT_ATOMIC_WRITE = 62725
_F2FS_IOC_GET_FEATURES = 2147808524
_F2FS_IOC_GET_FEATURES = 2147808524 // -2147158772
_F2FS_FEATURE_ATOMIC_WRITE = 4
)

View File

@ -1,4 +1,4 @@
//go:build !linux || !(amd64 || arm64 || riscv64 || ppc64le) || sqlite3_nosys
//go:build !linux || !(amd64 || arm64 || riscv64) || sqlite3_nosys
package vfs

View File

@ -1,4 +1,4 @@
//go:build (darwin || linux) && (amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_flock || sqlite3_noshm || sqlite3_nosys)
//go:build (darwin || linux) && (386 || arm || amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_flock || sqlite3_noshm || sqlite3_nosys)
package vfs
@ -6,11 +6,13 @@ import (
"context"
"io"
"os"
"sync"
"time"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/tetratelabs/wazero/api"
"golang.org/x/sys/unix"
"github.com/ncruces/go-sqlite3/internal/util"
)
// SupportsSharedMemory is false on platforms that do not support shared memory.
@ -45,12 +47,15 @@ func NewSharedMemory(path string, flags OpenFlag) SharedMemory {
}
}
var _ blockingSharedMemory = &vfsShm{}
type vfsShm struct {
*os.File
path string
regions []*util.MappedRegion
readOnly bool
blocking bool
sync.Mutex
}
func (s *vfsShm) shmOpen() _ErrorCode {
@ -196,6 +201,12 @@ func (s *vfsShm) shmUnmap(delete bool) {
s.File = nil
}
func (s *vfsShm) shmBarrier() {
s.Lock()
//lint:ignore SA2001 memory barrier.
s.Unlock()
}
func (s *vfsShm) shmEnableBlocking(block bool) {
s.blocking = block
}

View File

@ -1,4 +1,4 @@
//go:build (freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && (amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_noshm || sqlite3_nosys)
//go:build (freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && (386 || arm || amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_noshm || sqlite3_nosys)
package vfs
@ -8,9 +8,10 @@ import (
"os"
"sync"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/tetratelabs/wazero/api"
"golang.org/x/sys/unix"
"github.com/ncruces/go-sqlite3/internal/util"
)
// SupportsSharedMemory is false on platforms that do not support shared memory.
@ -269,3 +270,9 @@ func (s *vfsShm) shmUnmap(delete bool) {
}
s.Close()
}
func (s *vfsShm) shmBarrier() {
s.lockMtx.Lock()
//lint:ignore SA2001 memory barrier.
s.lockMtx.Unlock()
}

View File

@ -1,4 +1,4 @@
//go:build !(darwin || linux || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) || !(amd64 || arm64 || riscv64 || ppc64le) || sqlite3_noshm || sqlite3_nosys
//go:build !(darwin || linux || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) || !(386 || arm || amd64 || arm64 || riscv64 || ppc64le) || sqlite3_noshm || sqlite3_nosys
package vfs

View File

@ -5,13 +5,15 @@ import (
"crypto/rand"
"io"
"reflect"
"sync"
"strings"
"time"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/ncruces/julianday"
"github.com/tetratelabs/wazero"
"github.com/tetratelabs/wazero/api"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/ncruces/go-sqlite3/util/sql3util"
"github.com/ncruces/julianday"
)
// ExportHostFunctions is an internal API users need not call directly.
@ -146,7 +148,7 @@ func vfsOpen(ctx context.Context, mod api.Module, pVfs, zPath, pFile uint32, fla
}
if file, ok := file.(FilePowersafeOverwrite); ok {
if b, ok := util.ParseBool(name.URIParameter("psow")); ok {
if b, ok := sql3util.ParseBool(name.URIParameter("psow")); ok {
file.SetPowersafeOverwrite(b)
}
}
@ -157,6 +159,7 @@ func vfsOpen(ctx context.Context, mod api.Module, pVfs, zPath, pFile uint32, fla
if pOutFlags != 0 {
util.WriteUint32(mod, pOutFlags, uint32(flags))
}
file = cksmWrapFile(name, flags, file)
vfsFileRegister(ctx, mod, pFile, file)
return _OK
}
@ -235,20 +238,19 @@ func vfsCheckReservedLock(ctx context.Context, mod api.Module, pFile, pResOut ui
func vfsFileControl(ctx context.Context, mod api.Module, pFile uint32, op _FcntlOpcode, pArg uint32) _ErrorCode {
file := vfsFileGet(ctx, mod, pFile).(File)
if file, ok := file.(fileControl); ok {
return file.fileControl(ctx, mod, op, pArg)
}
return vfsFileControlImpl(ctx, mod, file, op, pArg)
}
func vfsFileControlImpl(ctx context.Context, mod api.Module, file File, op _FcntlOpcode, pArg uint32) _ErrorCode {
switch op {
case _FCNTL_LOCKSTATE:
if file, ok := file.(FileLockState); ok {
util.WriteUint32(mod, pArg, uint32(file.LockState()))
return _OK
}
case _FCNTL_LOCK_TIMEOUT:
if file, ok := file.(FileSharedMemory); ok {
if iface, ok := file.SharedMemory().(interface{ shmEnableBlocking(bool) }); ok {
if i := util.ReadUint32(mod, pArg); i == 0 || i == 1 {
iface.shmEnableBlocking(i != 0)
}
if lk := file.LockState(); lk <= LOCK_EXCLUSIVE {
util.WriteUint32(mod, pArg, uint32(lk))
return _OK
}
}
@ -329,15 +331,15 @@ func vfsFileControl(ctx context.Context, mod api.Module, pFile uint32, op _Fcntl
return vfsErrorCode(err, _IOERR_ROLLBACK_ATOMIC)
}
case _FCNTL_CKPT_DONE:
if file, ok := file.(FileCheckpoint); ok {
err := file.CheckpointDone()
return vfsErrorCode(err, _IOERR)
}
case _FCNTL_CKPT_START:
if file, ok := file.(FileCheckpoint); ok {
err := file.CheckpointStart()
return vfsErrorCode(err, _IOERR)
file.CheckpointStart()
return _OK
}
case _FCNTL_CKPT_DONE:
if file, ok := file.(FileCheckpoint); ok {
file.CheckpointDone()
return _OK
}
case _FCNTL_PRAGMA:
@ -349,7 +351,7 @@ func vfsFileControl(ctx context.Context, mod api.Module, pFile uint32, op _Fcntl
value = util.ReadString(mod, ptr, _MAX_SQL_LENGTH)
}
out, err := file.Pragma(name, value)
out, err := file.Pragma(strings.ToLower(name), value)
ret := vfsErrorCode(err, _ERROR)
if ret == _ERROR {
@ -366,6 +368,14 @@ func vfsFileControl(ctx context.Context, mod api.Module, pFile uint32, op _Fcntl
}
return ret
}
case _FCNTL_LOCK_TIMEOUT:
if file, ok := file.(FileSharedMemory); ok {
if shm, ok := file.SharedMemory().(blockingSharedMemory); ok {
shm.shmEnableBlocking(util.ReadUint32(mod, pArg) != 0)
return _OK
}
}
}
// Consider also implementing these opcodes (in use by SQLite):
@ -385,11 +395,9 @@ func vfsDeviceCharacteristics(ctx context.Context, mod api.Module, pFile uint32)
return file.DeviceCharacteristics()
}
var shmBarrier sync.Mutex
func vfsShmBarrier(ctx context.Context, mod api.Module, pFile uint32) {
shmBarrier.Lock()
defer shmBarrier.Unlock()
shm := vfsFileGet(ctx, mod, pFile).(FileSharedMemory).SharedMemory()
shm.shmBarrier()
}
func vfsShmMap(ctx context.Context, mod api.Module, pFile uint32, iRegion, szRegion int32, bExtend, pp uint32) _ErrorCode {