[performance] massively improved ActivityPub delivery worker efficiency (#2812)

* add delivery worker type that pulls from queue to httpclient package

* finish up some code commenting, bodge a vendored activity library change, integrate the deliverypool changes into transportcontroller

* hook up queue deletion logic

* support deleting queued http requests by target ID

* don't index APRequest by hostname in the queue

* use gorun

* use the original context's values when wrapping msg type as delivery{}

* actually log in the AP delivery worker ...

* add uncommitted changes

* use errors.AsV2()

* use errorsv2.AsV2()

* finish adding some code comments, add bad host handling to delivery workers

* slightly tweak deliveryworkerpool API, use advanced sender multiplier

* remove PopCtx() method, let others instead rely on Wait()

* shuffle things around to move delivery stuff into transport/ subpkg

* remove dead code

* formatting

* validate request before queueing for delivery

* finish adding code comments, fix up backoff code

* finish adding more code comments

* clamp minimum no. senders to 1

* add start/stop logging to delivery worker, some slight changes

* remove double logging

* use worker ptrs

* expose the embedded log fields in httpclient.Request{}

* ensure request context values are preserved when updating ctx

* add delivery worker tests

* fix linter issues

* ensure delivery worker gets inited in testrig

* fix tests to delivering messages to check worker delivery queue

* update error type to use ptr instead of value receiver

* fix test calling Workers{}.Start() instead of testrig.StartWorkers()

* update docs for advanced-sender-multiplier

* update to the latest activity library version

* add comment about not using httptest.Server{}

internal/transport/controller.go

@@ -28,7 +28,6 @@ import (
 	"io"
 	"net/http"
 	"net/url"
-	"runtime"
 
 	"codeberg.org/gruf/go-byteutil"
 	"codeberg.org/gruf/go-cache/v3"
@@ -56,24 +55,16 @@ type controller struct {
 	client    pub.HttpClient
 	trspCache cache.TTLCache[string, *transport]
 	userAgent string
-	senders   int // no. concurrent batch delivery routines.
 }
 
 // NewController returns an implementation of the Controller interface for creating new transports
 func NewController(state *state.State, federatingDB federatingdb.DB, clock pub.Clock, client pub.HttpClient) Controller {
 	var (
-		host             = config.GetHost()
-		proto            = config.GetProtocol()
-		version          = config.GetSoftwareVersion()
-		senderMultiplier = config.GetAdvancedSenderMultiplier()
+		host    = config.GetHost()
+		proto   = config.GetProtocol()
+		version = config.GetSoftwareVersion()
 	)
 
-	senders := senderMultiplier * runtime.GOMAXPROCS(0)
-	if senders < 1 {
-		// Clamp senders to 1.
-		senders = 1
-	}
-
 	c := &controller{
 		state:     state,
 		fedDB:     federatingDB,
@@ -81,7 +72,6 @@ func NewController(state *state.State, federatingDB federatingdb.DB, clock pub.C
 		client:    client,
 		trspCache: cache.NewTTL[string, *transport](0, 100, 0),
 		userAgent: fmt.Sprintf("gotosocial/%s (+%s://%s)", version, proto, host),
-		senders:   senders,
 	}
 
 	return c