[chore] Recompile wasm every 500 goes

internal/media/ffmpeg/ffmpeg.go

@@ -19,22 +19,121 @@ package ffmpeg
 
 import (
 	"context"
+	"sync"
+
+	ffmpeglib "codeberg.org/gruf/go-ffmpreg/embed/ffmpeg"
+	"github.com/superseriousbusiness/gotosocial/internal/log"
+	"github.com/tetratelabs/wazero"
+	"github.com/tetratelabs/wazero/sys"
 )
 
-// ffmpegRunner limits the number of
+var (
-// ffmpeg WebAssembly instances that
+	// ffmpegRunner limits the number of
-// may be concurrently running, in
+	// ffmpeg WebAssembly instances that
-// order to reduce memory usage.
+	// may be concurrently running, in
-var ffmpegRunner runner
+	// order to reduce memory usage.
+	ffmpegRunner runner
+
+	// ffmpeg compiled WASM.
+	ffmpeg wazero.CompiledModule
+
+	// Number of times ffmpeg
+	// compiled WASM has run.
+	ffmpegRunCount int
+
+	// Sync for updating run count
+	// and recompiling ffmpeg.
+	ffmpegM sync.Mutex
+)
 
 // InitFfmpeg precompiles the ffmpeg WebAssembly source into memory and
 // prepares the runner to only allow max given concurrent running instances.
 func InitFfmpeg(ctx context.Context, max int) error {
+
+	// Ensure runner initialized.
 	ffmpegRunner.Init(max)
-	return compileFfmpeg(ctx)
+
+	// Ensure runtime initialized.
+	if err := initRuntime(ctx); err != nil {
+		return err
+	}
+
+	// Ensure ffmpeg compiled.
+	if ffmpeg == nil {
+		return compileFfmpeg(ctx)
+	}
+
+	return nil
+}
+
+// compileFfmpeg ensures the ffmpeg WebAssembly
+// module has been pre-compiled into memory.
+func compileFfmpeg(ctx context.Context) error {
+	var err error
+	ffmpeg, err = runtime.CompileModule(ctx, ffmpeglib.B)
+	return err
 }
 
 // Ffmpeg runs the given arguments with an instance of ffmpeg.
 func Ffmpeg(ctx context.Context, args Args) (uint32, error) {
-	return ffmpegRunner.Run(ctx, ffmpeg, args)
+	return ffmpegRunner.Run(ctx, func() (uint32, error) {
+
+		// Update run count + check if we
+		// need to recompile the module.
+		ffmpegM.Lock()
+		{
+			ffmpegRunCount++
+			if ffmpegRunCount > 500 {
+				// Over our threshold of runs, close
+				// current compiled module and recompile.
+				if err := ffmpeg.Close(ctx); err != nil {
+					ffmpegM.Unlock()
+					return 0, err
+				}
+
+				if err := compileFfmpeg(ctx); err != nil {
+					ffmpegM.Unlock()
+					return 0, err
+				}
+
+				ffmpegRunCount = 0
+			}
+		}
+		ffmpegM.Unlock()
+
+		// Prefix module name as argv0 to args.
+		cargs := make([]string, len(args.Args)+1)
+		copy(cargs[1:], args.Args)
+		cargs[0] = "ffmpeg"
+
+		// Create base module config.
+		modcfg := wazero.NewModuleConfig()
+		modcfg = modcfg.WithArgs(cargs...)
+		modcfg = modcfg.WithStdin(args.Stdin)
+		modcfg = modcfg.WithStdout(args.Stdout)
+		modcfg = modcfg.WithStderr(args.Stderr)
+
+		if args.Config != nil {
+			// Pass through config fn.
+			modcfg = args.Config(modcfg)
+		}
+
+		// Instantiate the module from precompiled wasm module data.
+		mod, err := runtime.InstantiateModule(ctx, ffmpeg, modcfg)
+
+		if mod != nil {
+			// Ensure closed.
+			if err := mod.Close(ctx); err != nil {
+				log.Errorf(ctx, "error closing: %v", err)
+			}
+		}
+
+		// Try extract exit code.
+		switch err := err.(type) {
+		case *sys.ExitError:
+			return err.ExitCode(), nil
+		default:
+			return 0, err
+		}
+	})
 }

internal/media/ffmpeg/ffprobe.go

@@ -19,22 +19,121 @@ package ffmpeg
 
 import (
 	"context"
+	"sync"
+
+	ffprobelib "codeberg.org/gruf/go-ffmpreg/embed/ffprobe"
+	"github.com/superseriousbusiness/gotosocial/internal/log"
+	"github.com/tetratelabs/wazero"
+	"github.com/tetratelabs/wazero/sys"
 )
 
-// ffprobeRunner limits the number of
+var (
-// ffprobe WebAssembly instances that
+	// ffprobeRunner limits the number of
-// may be concurrently running, in
+	// ffprobe WebAssembly instances that
-// order to reduce memory usage.
+	// may be concurrently running, in
-var ffprobeRunner runner
+	// order to reduce memory usage.
+	ffprobeRunner runner
+
+	// ffprobe compiled WASM.
+	ffprobe wazero.CompiledModule
+
+	// Number of times ffprobe
+	// compiled WASM has run.
+	ffprobeRunCount int
+
+	// Sync for updating run count
+	// and recompiling ffprobe.
+	ffprobeM sync.Mutex
+)
 
 // InitFfprobe precompiles the ffprobe WebAssembly source into memory and
 // prepares the runner to only allow max given concurrent running instances.
 func InitFfprobe(ctx context.Context, max int) error {
+
+	// Ensure runner initialized.
 	ffprobeRunner.Init(max)
-	return compileFfprobe(ctx)
+
+	// Ensure runtime initialized.
+	if err := initRuntime(ctx); err != nil {
+		return err
+	}
+
+	// Ensure ffprobe compiled.
+	if ffprobe == nil {
+		return compileFfprobe(ctx)
+	}
+
+	return nil
+}
+
+// compileFfprobe ensures the ffprobe WebAssembly
+// module has been pre-compiled into memory.
+func compileFfprobe(ctx context.Context) error {
+	var err error
+	ffprobe, err = runtime.CompileModule(ctx, ffprobelib.B)
+	return err
 }
 
 // Ffprobe runs the given arguments with an instance of ffprobe.
 func Ffprobe(ctx context.Context, args Args) (uint32, error) {
-	return ffprobeRunner.Run(ctx, ffprobe, args)
+	return ffprobeRunner.Run(ctx, func() (uint32, error) {
+
+		// Update run count + check if we
+		// need to recompile the module.
+		ffprobeM.Lock()
+		{
+			ffprobeRunCount++
+			if ffprobeRunCount > 500 {
+				// Over our threshold of runs, close
+				// current compiled module and recompile.
+				if err := ffprobe.Close(ctx); err != nil {
+					ffprobeM.Unlock()
+					return 0, err
+				}
+
+				if err := compileFfprobe(ctx); err != nil {
+					ffprobeM.Unlock()
+					return 0, err
+				}
+
+				ffprobeRunCount = 0
+			}
+		}
+		ffprobeM.Unlock()
+
+		// Prefix module name as argv0 to args.
+		cargs := make([]string, len(args.Args)+1)
+		copy(cargs[1:], args.Args)
+		cargs[0] = "ffprobe"
+
+		// Create base module config.
+		modcfg := wazero.NewModuleConfig()
+		modcfg = modcfg.WithArgs(cargs...)
+		modcfg = modcfg.WithStdin(args.Stdin)
+		modcfg = modcfg.WithStdout(args.Stdout)
+		modcfg = modcfg.WithStderr(args.Stderr)
+
+		if args.Config != nil {
+			// Pass through config fn.
+			modcfg = args.Config(modcfg)
+		}
+
+		// Instantiate the module from precompiled wasm module data.
+		mod, err := runtime.InstantiateModule(ctx, ffprobe, modcfg)
+
+		if mod != nil {
+			// Ensure closed.
+			if err := mod.Close(ctx); err != nil {
+				log.Errorf(ctx, "error closing: %v", err)
+			}
+		}
+
+		// Try extract exit code.
+		switch err := err.(type) {
+		case *sys.ExitError:
+			return err.ExitCode(), nil
+		default:
+			return 0, err
+		}
+	})
 }

internal/media/ffmpeg/runner.go

@@ -19,8 +19,6 @@ package ffmpeg
 
 import (
 	"context"
-
-	"github.com/tetratelabs/wazero"
 )
 
 // runner simply abstracts away the complexities
@@ -50,9 +48,12 @@ func (r *runner) Init(n int) {
 	}
 }
 
-// Run will attempt to pass the given compiled WebAssembly module with args to run(), waiting on
+// Run will instantiate (run) the given
-// the receiving runner until a free slot is available to run an instance, (if a limit is enabled).
+// function once a free slot is available.
-func (r *runner) Run(ctx context.Context, cmod wazero.CompiledModule, args Args) (uint32, error) {
+func (r *runner) Run(
+	ctx context.Context,
+	run func() (uint32, error),
+) (uint32, error) {
 	select {
 	// Context canceled.
 	case <-ctx.Done():
@@ -65,6 +66,6 @@ func (r *runner) Run(ctx context.Context, cmod wazero.CompiledModule, args Args)
 	// Release slot back to pool on end.
 	defer func() { r.pool <- struct{}{} }()
 
-	// Pass to main module runner.
+	// Run provided function.
-	return run(ctx, cmod, args)
+	return run()
 }

internal/media/ffmpeg/runtime.go

@@ -26,7 +26,6 @@ import (
 	ffprobelib "codeberg.org/gruf/go-ffmpreg/embed/ffprobe"
 	"github.com/tetratelabs/wazero"
 	"github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1"
-	"github.com/tetratelabs/wazero/sys"
 )
 
 // Use all core features required by ffmpeg / ffprobe
@@ -34,15 +33,6 @@ import (
 const corefeatures = ffprobelib.CoreFeatures |
 	ffmpeglib.CoreFeatures
 
-var (
-	// shared WASM runtime instance.
-	runtime wazero.Runtime
-
-	// ffmpeg / ffprobe compiled WASM.
-	ffmpeg  wazero.CompiledModule
-	ffprobe wazero.CompiledModule
-)
-
 // Args encapsulates the passing of common
 // configuration options to run an instance
 // of a compiled WebAssembly module that is
@@ -62,96 +52,8 @@ type Args struct {
 	Args []string
 }
 
-// run will run the given compiled
+// shared WASM runtime instance.
-// WebAssembly module using given args,
+var runtime wazero.Runtime
-// using the global wazero runtime.
-func run(
-	ctx context.Context,
-	cmod wazero.CompiledModule,
-	args Args,
-) (
-	uint32, // exit code
-	error,
-) {
-	// Prefix module name as argv0 to args.
-	cargs := make([]string, len(args.Args)+1)
-	copy(cargs[1:], args.Args)
-	cargs[0] = cmod.Name()
-
-	// Create base module config.
-	modcfg := wazero.NewModuleConfig()
-	modcfg = modcfg.WithArgs(cargs...)
-	modcfg = modcfg.WithStdin(args.Stdin)
-	modcfg = modcfg.WithStdout(args.Stdout)
-	modcfg = modcfg.WithStderr(args.Stderr)
-
-	if args.Config != nil {
-		// Pass through config fn.
-		modcfg = args.Config(modcfg)
-	}
-
-	// Instantiate the module from precompiled wasm module data.
-	mod, err := runtime.InstantiateModule(ctx, cmod, modcfg)
-
-	if mod != nil {
-		// Ensure closed.
-		_ = mod.Close(ctx)
-	}
-
-	// Try extract exit code.
-	switch err := err.(type) {
-	case *sys.ExitError:
-		return err.ExitCode(), nil
-	default:
-		return 0, err
-	}
-}
-
-// compileFfmpeg ensures the ffmpeg WebAssembly has been
-// pre-compiled into memory. If already compiled is a no-op.
-func compileFfmpeg(ctx context.Context) error {
-	if ffmpeg != nil {
-		return nil
-	}
-
-	// Ensure runtime already initialized.
-	if err := initRuntime(ctx); err != nil {
-		return err
-	}
-
-	// Compile the ffmpeg WebAssembly module into memory.
-	cmod, err := runtime.CompileModule(ctx, ffmpeglib.B)
-	if err != nil {
-		return err
-	}
-
-	// Set module.
-	ffmpeg = cmod
-	return nil
-}
-
-// compileFfprobe ensures the ffprobe WebAssembly has been
-// pre-compiled into memory. If already compiled is a no-op.
-func compileFfprobe(ctx context.Context) error {
-	if ffprobe != nil {
-		return nil
-	}
-
-	// Ensure runtime already initialized.
-	if err := initRuntime(ctx); err != nil {
-		return err
-	}
-
-	// Compile the ffprobe WebAssembly module into memory.
-	cmod, err := runtime.CompileModule(ctx, ffprobelib.B)
-	if err != nil {
-		return err
-	}
-
-	// Set module.
-	ffprobe = cmod
-	return nil
-}
 
 // initRuntime initializes the global wazero.Runtime,
 // if already initialized this function is a no-op.