[performance] improved request batching (removes need for queueing) (#1687)

* revamp http client to not limit requests, instead use sender worker

Signed-off-by: kim <grufwub@gmail.com>

* remove separate sender worker pool, spawn 2*GOMAXPROCS batch senders each time, no need for transport cache sweeping

Signed-off-by: kim <grufwub@gmail.com>

* improve batch senders to keep popping recipients until remote URL found

Signed-off-by: kim <grufwub@gmail.com>

* fix recipient looping issue

Signed-off-by: kim <grufwub@gmail.com>

* fix missing mutex unlock

Signed-off-by: kim <grufwub@gmail.com>

* move request id ctx key to gtscontext, finish filling out more code comments, add basic support for not logging client IP

Signed-off-by: kim <grufwub@gmail.com>

* slight code reformatting

Signed-off-by: kim <grufwub@gmail.com>

* a whitespace

Signed-off-by: kim <grufwub@gmail.com>

* remove unused code

Signed-off-by: kim <grufwub@gmail.com>

* add missing license headers

Signed-off-by: kim <grufwub@gmail.com>

* fix request backoff calculation

Signed-off-by: kim <grufwub@gmail.com>

---------

Signed-off-by: kim <grufwub@gmail.com>

internal/httpclient/client_test.go

@@ -48,14 +48,6 @@ var bodies = []string{
 	"body with\r\nnewlines",
 }
 
-// Note:
-// There is no test for the .MaxOpenConns implementation
-// in the httpclient.Client{}, due to the difficult to test
-// this. The block is only held for the actual dial out to
-// the connection, so the usual test of blocking and holding
-// open this queue slot to check we can't open another isn't
-// an easy test here.
-
 func TestHTTPClientSmallBody(t *testing.T) {
 	for _, body := range bodies {
 		_TestHTTPClientWithBody(t, []byte(body), int(^uint16(0)))