[performance] processing media and scheduled jobs improvements (#1482)

* replace media workers with just runners.WorkerPool, move to state structure, use go-sched for global task scheduling

* improved code comment

* fix worker tryUntil function, update go-runners/go-sched

* make preprocess functions package public, use these where possible to stop doubled up processing

* remove separate emoji worker pool

* limit calls to time.Now() during media preprocessing

* use Processor{} to manage singular runtime of processing media

* ensure workers get started when media manager is used

* improved error setting in processing media, fix media test

* port changes from processingmedia to processing emoji

* finish code commenting

* finish code commenting and comment-out client API + federator worker pools until concurrency worker pools replaced

* linterrrrrrrrrrrrrrrr

---------

Signed-off-by: kim <grufwub@gmail.com>
This commit is contained in:
kim
2023-02-13 18:40:48 +00:00
committed by GitHub
parent 76d1b484d0
commit acc95923da
54 changed files with 1853 additions and 2680 deletions

View File

@@ -157,6 +157,34 @@ func (pool *WorkerPool) EnqueueCtx(ctx context.Context, fn WorkerFunc) bool {
}
}
// MustEnqueueCtx functionally performs similarly to WorkerPool.EnqueueCtx(), but in the case
// that the provided <-ctx.Done() is closed, it is passed asynchronously to WorkerPool.Enqueue().
// Return boolean indicates whether function was executed in time before <-ctx.Done() is closed.
func (pool *WorkerPool) MustEnqueueCtx(ctx context.Context, fn WorkerFunc) (ok bool) {
// Check valid fn
if fn == nil {
return false
}
select {
case <-ctx.Done():
// We failed to add this entry to the worker queue before the
// incoming context was cancelled. So to ensure processing
// we simply queue it asynchronously and return early to caller.
go pool.Enqueue(fn)
return false
case <-pool.svc.Done():
// Pool ctx cancelled
fn(closedctx)
return false
case pool.fns <- fn:
// Placed fn in queue
return true
}
}
// EnqueueNow attempts Enqueue but returns false if not executed.
func (pool *WorkerPool) EnqueueNow(fn WorkerFunc) bool {
// Check valid fn