[feature/OFFICIALLY UNSUPPORTED] add nowasm build tag to disable building with WebAssembly (#3429)

* add experimental build-tag 'nowasm' which uses local ffmpeg / ffprobe

* updated experimental support message

* add comment to build script explaining build tag

* add nowasm build tags to moderncsqlite files
This commit is contained in:
kim 2024-10-14 09:59:12 +00:00 committed by GitHub
parent 157ee3193d
commit 6a76b9d609
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 301 additions and 26 deletions

View File

@ -15,7 +15,7 @@
// You should have received a copy of the GNU Affero General Public License // You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>. // along with this program. If not, see <http://www.gnu.org/licenses/>.
//go:build !moderncsqlite3 //go:build !moderncsqlite3 && !nowasm
package sqlite package sqlite

View File

@ -15,7 +15,7 @@
// You should have received a copy of the GNU Affero General Public License // You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>. // along with this program. If not, see <http://www.gnu.org/licenses/>.
//go:build moderncsqlite3 //go:build moderncsqlite3 || nowasm
package sqlite package sqlite

View File

@ -15,7 +15,7 @@
// You should have received a copy of the GNU Affero General Public License // You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>. // along with this program. If not, see <http://www.gnu.org/licenses/>.
//go:build !moderncsqlite3 //go:build !moderncsqlite3 && !nowasm
package sqlite package sqlite

View File

@ -15,7 +15,7 @@
// You should have received a copy of the GNU Affero General Public License // You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>. // along with this program. If not, see <http://www.gnu.org/licenses/>.
//go:build moderncsqlite3 //go:build moderncsqlite3 || nowasm
package sqlite package sqlite

View File

@ -0,0 +1,28 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ffmpeg
import (
"codeberg.org/gruf/go-ffmpreg/wasm"
)
// Args encapsulates the passing of common
// configuration options to run an instance
// of a compiled WebAssembly module that is
// run in a typical CLI manner.
type Args = wasm.Args

View File

@ -0,0 +1,142 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//go:build nowasm
package ffmpeg
import (
"context"
"fmt"
"io"
"io/fs"
"os/exec"
"codeberg.org/gruf/go-ffmpreg/wasm"
"github.com/tetratelabs/wazero"
"github.com/tetratelabs/wazero/sys"
)
func init() {
fmt.Println("!! you are using an unsupported build configuration of gotosocial with WebAssembly disabled !!")
fmt.Println("!! please do not file bug reports regarding media processing with this configuration !!")
fmt.Println("!! it is also less secure; this does not enforce version checks on ffmpeg / ffprobe versions !!")
}
// runCmd will run 'name' with the given arguments, returning exit code or error.
func runCmd(ctx context.Context, name string, args wasm.Args) (uint32, error) {
cmd := exec.CommandContext(ctx, name, args.Args...) //nolint:gosec
// Set provided std files.
cmd.Stdin = args.Stdin
cmd.Stdout = args.Stdout
cmd.Stderr = args.Stderr
if args.Config != nil {
// Gather some information
// from module config func.
var cfg falseModuleConfig
_ = args.Config(&cfg)
// Extract from conf.
cmd.Env = cfg.env
}
// Run prepared command, catching err type.
switch err := cmd.Run(); err := err.(type) {
// Extract code from
// any exit error type.
case *exec.ExitError:
rc := err.ExitCode()
return uint32(rc), err
default:
return 0, err
}
}
type falseModuleConfig struct{ env []string }
func (cfg *falseModuleConfig) WithArgs(...string) wazero.ModuleConfig {
return cfg // noop
}
func (cfg *falseModuleConfig) WithEnv(key string, value string) wazero.ModuleConfig {
cfg.env = append(cfg.env, key+"="+value)
return cfg // noop
}
func (cfg *falseModuleConfig) WithFS(fs.FS) wazero.ModuleConfig {
return cfg // noop
}
func (cfg *falseModuleConfig) WithFSConfig(wazero.FSConfig) wazero.ModuleConfig {
return cfg // noop
}
func (cfg *falseModuleConfig) WithName(string) wazero.ModuleConfig {
return cfg // noop
}
func (cfg *falseModuleConfig) WithStartFunctions(...string) wazero.ModuleConfig {
return cfg // noop
}
func (cfg *falseModuleConfig) WithStderr(io.Writer) wazero.ModuleConfig {
return cfg // noop
}
func (cfg *falseModuleConfig) WithStdin(io.Reader) wazero.ModuleConfig {
return cfg // noop
}
func (cfg *falseModuleConfig) WithStdout(io.Writer) wazero.ModuleConfig {
return cfg // noop
}
func (cfg *falseModuleConfig) WithWalltime(sys.Walltime, sys.ClockResolution) wazero.ModuleConfig {
return cfg // noop
}
func (cfg *falseModuleConfig) WithSysWalltime() wazero.ModuleConfig {
return cfg // noop
}
func (cfg *falseModuleConfig) WithNanotime(sys.Nanotime, sys.ClockResolution) wazero.ModuleConfig {
return cfg // noop
}
func (cfg *falseModuleConfig) WithSysNanotime() wazero.ModuleConfig {
return cfg // noop
}
func (cfg *falseModuleConfig) WithNanosleep(sys.Nanosleep) wazero.ModuleConfig {
return cfg // noop
}
func (cfg *falseModuleConfig) WithOsyield(sys.Osyield) wazero.ModuleConfig {
return cfg // noop
}
func (cfg *falseModuleConfig) WithSysNanosleep() wazero.ModuleConfig {
return cfg // noop
}
func (cfg *falseModuleConfig) WithRandSource(io.Reader) wazero.ModuleConfig {
return cfg // noop
}

View File

@ -15,10 +15,14 @@
// You should have received a copy of the GNU Affero General Public License // You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>. // along with this program. If not, see <http://www.gnu.org/licenses/>.
//go:build !nowasm
package ffmpeg package ffmpeg
import ( import (
"context" "context"
"codeberg.org/gruf/go-ffmpreg/wasm"
) )
// ffmpegRunner limits the number of // ffmpegRunner limits the number of
@ -36,5 +40,7 @@ func InitFfmpeg(ctx context.Context, max int) error {
// Ffmpeg runs the given arguments with an instance of ffmpeg. // Ffmpeg runs the given arguments with an instance of ffmpeg.
func Ffmpeg(ctx context.Context, args Args) (uint32, error) { func Ffmpeg(ctx context.Context, args Args) (uint32, error) {
return ffmpegRunner.Run(ctx, ffmpeg, args) return ffmpegRunner.Run(ctx, func() (uint32, error) {
return wasm.Run(ctx, runtime, ffmpeg, args)
})
} }

View File

@ -0,0 +1,49 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//go:build nowasm
package ffmpeg
import (
"context"
"os/exec"
)
// ffmpegRunner limits the number of
// ffmpeg WebAssembly instances that
// may be concurrently running, in
// order to reduce memory usage.
var ffmpegRunner runner
// InitFfmpeg looks for a local copy of ffmpeg in path, and prepares
// the runner to only allow max given concurrent running instances.
func InitFfmpeg(ctx context.Context, max int) error {
_, err := exec.LookPath("ffmpeg")
if err != nil {
return err
}
ffmpegRunner.Init(max)
return nil
}
// Ffmpeg runs the given arguments with an instance of ffmpeg.
func Ffmpeg(ctx context.Context, args Args) (uint32, error) {
return ffmpegRunner.Run(ctx, func() (uint32, error) {
return runCmd(ctx, "ffmpeg", args)
})
}

View File

@ -15,10 +15,14 @@
// You should have received a copy of the GNU Affero General Public License // You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>. // along with this program. If not, see <http://www.gnu.org/licenses/>.
//go:build !nowasm
package ffmpeg package ffmpeg
import ( import (
"context" "context"
"codeberg.org/gruf/go-ffmpreg/wasm"
) )
// ffprobeRunner limits the number of // ffprobeRunner limits the number of
@ -36,5 +40,7 @@ func InitFfprobe(ctx context.Context, max int) error {
// Ffprobe runs the given arguments with an instance of ffprobe. // Ffprobe runs the given arguments with an instance of ffprobe.
func Ffprobe(ctx context.Context, args Args) (uint32, error) { func Ffprobe(ctx context.Context, args Args) (uint32, error) {
return ffprobeRunner.Run(ctx, ffprobe, args) return ffmpegRunner.Run(ctx, func() (uint32, error) {
return wasm.Run(ctx, runtime, ffprobe, args)
})
} }

View File

@ -0,0 +1,49 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//go:build nowasm
package ffmpeg
import (
"context"
"os/exec"
)
// ffprobeRunner limits the number of
// ffprobe WebAssembly instances that
// may be concurrently running, in
// order to reduce memory usage.
var ffprobeRunner runner
// InitFfprobe looks for a local copy of ffprobe in path, and prepares
// the runner to only allow max given concurrent running instances.
func InitFfprobe(ctx context.Context, max int) error {
_, err := exec.LookPath("ffprobe")
if err != nil {
return err
}
ffprobeRunner.Init(max)
return nil
}
// Ffprobe runs the given arguments with an instance of ffprobe.
func Ffprobe(ctx context.Context, args Args) (uint32, error) {
return ffprobeRunner.Run(ctx, func() (uint32, error) {
return runCmd(ctx, "ffprobe", args)
})
}

View File

@ -19,9 +19,6 @@ package ffmpeg
import ( import (
"context" "context"
"codeberg.org/gruf/go-ffmpreg/wasm"
"github.com/tetratelabs/wazero"
) )
// runner simply abstracts away the complexities // runner simply abstracts away the complexities
@ -53,7 +50,7 @@ func (r *runner) Init(n int) {
// Run will attempt to pass the given compiled WebAssembly module with args to run(), waiting on // Run will attempt to pass the given compiled WebAssembly module with args to run(), waiting on
// the receiving runner until a free slot is available to run an instance, (if a limit is enabled). // the receiving runner until a free slot is available to run an instance, (if a limit is enabled).
func (r *runner) Run(ctx context.Context, cmod wazero.CompiledModule, args Args) (uint32, error) { func (r *runner) Run(ctx context.Context, run func() (uint32, error)) (uint32, error) {
select { select {
// Context canceled. // Context canceled.
case <-ctx.Done(): case <-ctx.Done():
@ -66,6 +63,6 @@ func (r *runner) Run(ctx context.Context, cmod wazero.CompiledModule, args Args)
// Release slot back to pool on end. // Release slot back to pool on end.
defer func() { r.pool <- struct{}{} }() defer func() { r.pool <- struct{}{} }()
// Pass to main module runner function. // Call run.
return wasm.Run(ctx, runtime, cmod, args) return run()
} }

View File

@ -15,6 +15,8 @@
// You should have received a copy of the GNU Affero General Public License // You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>. // along with this program. If not, see <http://www.gnu.org/licenses/>.
//go:build !nowasm
package ffmpeg package ffmpeg
import ( import (
@ -41,12 +43,6 @@ var (
ffprobe wazero.CompiledModule ffprobe wazero.CompiledModule
) )
// Args encapsulates the passing of common
// configuration options to run an instance
// of a compiled WebAssembly module that is
// run in a typical CLI manner.
type Args = wasm.Args
// compileFfmpeg ensures the ffmpeg WebAssembly has been // compileFfmpeg ensures the ffmpeg WebAssembly has been
// pre-compiled into memory. If already compiled is a no-op. // pre-compiled into memory. If already compiled is a no-op.
func compileFfmpeg(ctx context.Context) error { func compileFfmpeg(ctx context.Context) error {

View File

@ -15,14 +15,16 @@ GO_GCFLAGS=${GO_GCFLAGS-}
GO_BUILDTAGS="${GO_BUILDTAGS} debugenv" GO_BUILDTAGS="${GO_BUILDTAGS} debugenv"
# Available Go build tags, with explanation, followed by benefits of enabling it: # Available Go build tags, with explanation, followed by benefits of enabling it:
# - kvformat: enables prettier output of log fields (slightly better performance) # - kvformat: enables prettier output of log fields (slightly better performance)
# - timetzdata: embed timezone database inside binary (allow setting local time inside Docker containers, at cost of 450KB) # - timetzdata: embed timezone database inside binary (allow setting local time inside Docker containers, at cost of 450KB)
# - notracing: disables compiling-in otel tracing support (reduced binary size, better performance) # - notracing: disables compiling-in otel tracing support (reduced binary size, better performance)
# - nometrics: disables compiling-in otel metrics support (reduced binary size, better performance) # - nometrics: disables compiling-in otel metrics support (reduced binary size, better performance)
# - noerrcaller: disables caller function prefix in errors (slightly better performance, at cost of err readability) # - noerrcaller: disables caller function prefix in errors (slightly better performance, at cost of err readability)
# - debug: enables /debug/pprof endpoint (adds debug, at performance cost) # - debug: enables /debug/pprof endpoint (adds debug, at performance cost)
# - debugenv: enables /debug/pprof endpoint if DEBUG=1 env during runtime (adds debug, at performance cost) # - debugenv: enables /debug/pprof endpoint if DEBUG=1 env during runtime (adds debug, at performance cost)
# - moderncsqlite3: reverts to using the C-to-Go transpiled SQLite driver (disables the WASM-based SQLite driver) # - moderncsqlite3: reverts to using the C-to-Go transpiled SQLite driver (disables the WASM-based SQLite driver)
# - nowasm: [UNSUPPORTED] removes all WebAssembly from builds including
# ffmpeg, ffprobe and SQLite (instead falling back to modernc).
log_exec env CGO_ENABLED=0 go build -trimpath -v \ log_exec env CGO_ENABLED=0 go build -trimpath -v \
-tags "${GO_BUILDTAGS}" \ -tags "${GO_BUILDTAGS}" \
-ldflags="${GO_LDFLAGS}" \ -ldflags="${GO_LDFLAGS}" \