GoToSocial/internal/httpclient/client_test.go

155 lines
3.9 KiB
Go
Raw Normal View History

[security] transport.Controller{} and transport.Transport{} security and performance improvements (#564) * cache transports in controller by privkey-generated pubkey, add retry logic to transport requests Signed-off-by: kim <grufwub@gmail.com> * update code comments, defer mutex unlocks Signed-off-by: kim <grufwub@gmail.com> * add count to 'performing request' log message Signed-off-by: kim <grufwub@gmail.com> * reduce repeated conversions of same url.URL object Signed-off-by: kim <grufwub@gmail.com> * move worker.Worker to concurrency subpackage, add WorkQueue type, limit transport http client use by WorkQueue Signed-off-by: kim <grufwub@gmail.com> * fix security advisories regarding max outgoing conns, max rsp body size - implemented by a new httpclient.Client{} that wraps an underlying client with a queue to limit connections, and limit reader wrapping a response body with a configured maximum size - update pub.HttpClient args passed around to be this new httpclient.Client{} Signed-off-by: kim <grufwub@gmail.com> * add httpclient tests, move ip validation to separate package + change mechanism Signed-off-by: kim <grufwub@gmail.com> * fix merge conflicts Signed-off-by: kim <grufwub@gmail.com> * use singular mutex in transport rather than separate signer mus Signed-off-by: kim <grufwub@gmail.com> * improved useragent string Signed-off-by: kim <grufwub@gmail.com> * add note regarding missing test Signed-off-by: kim <grufwub@gmail.com> * remove useragent field from transport (instead store in controller) Signed-off-by: kim <grufwub@gmail.com> * shutup linter Signed-off-by: kim <grufwub@gmail.com> * reset other signing headers on each loop iteration Signed-off-by: kim <grufwub@gmail.com> * respect request ctx during retry-backoff sleep period Signed-off-by: kim <grufwub@gmail.com> * use external pkg with docs explaining performance "hack" Signed-off-by: kim <grufwub@gmail.com> * use http package constants instead of string method literals Signed-off-by: kim <grufwub@gmail.com> * add license file headers Signed-off-by: kim <grufwub@gmail.com> * update code comment to match new func names Signed-off-by: kim <grufwub@gmail.com> * updates to user-agent string Signed-off-by: kim <grufwub@gmail.com> * update signed testrig models to fit with new transport logic (instead uses separate signer now) Signed-off-by: kim <grufwub@gmail.com> * fuck you linter Signed-off-by: kim <grufwub@gmail.com>
2022-05-15 11:16:43 +02:00
/*
GoToSocial
Copyright (C) 2021-2023 GoToSocial Authors admin@gotosocial.org
[security] transport.Controller{} and transport.Transport{} security and performance improvements (#564) * cache transports in controller by privkey-generated pubkey, add retry logic to transport requests Signed-off-by: kim <grufwub@gmail.com> * update code comments, defer mutex unlocks Signed-off-by: kim <grufwub@gmail.com> * add count to 'performing request' log message Signed-off-by: kim <grufwub@gmail.com> * reduce repeated conversions of same url.URL object Signed-off-by: kim <grufwub@gmail.com> * move worker.Worker to concurrency subpackage, add WorkQueue type, limit transport http client use by WorkQueue Signed-off-by: kim <grufwub@gmail.com> * fix security advisories regarding max outgoing conns, max rsp body size - implemented by a new httpclient.Client{} that wraps an underlying client with a queue to limit connections, and limit reader wrapping a response body with a configured maximum size - update pub.HttpClient args passed around to be this new httpclient.Client{} Signed-off-by: kim <grufwub@gmail.com> * add httpclient tests, move ip validation to separate package + change mechanism Signed-off-by: kim <grufwub@gmail.com> * fix merge conflicts Signed-off-by: kim <grufwub@gmail.com> * use singular mutex in transport rather than separate signer mus Signed-off-by: kim <grufwub@gmail.com> * improved useragent string Signed-off-by: kim <grufwub@gmail.com> * add note regarding missing test Signed-off-by: kim <grufwub@gmail.com> * remove useragent field from transport (instead store in controller) Signed-off-by: kim <grufwub@gmail.com> * shutup linter Signed-off-by: kim <grufwub@gmail.com> * reset other signing headers on each loop iteration Signed-off-by: kim <grufwub@gmail.com> * respect request ctx during retry-backoff sleep period Signed-off-by: kim <grufwub@gmail.com> * use external pkg with docs explaining performance "hack" Signed-off-by: kim <grufwub@gmail.com> * use http package constants instead of string method literals Signed-off-by: kim <grufwub@gmail.com> * add license file headers Signed-off-by: kim <grufwub@gmail.com> * update code comment to match new func names Signed-off-by: kim <grufwub@gmail.com> * updates to user-agent string Signed-off-by: kim <grufwub@gmail.com> * update signed testrig models to fit with new transport logic (instead uses separate signer now) Signed-off-by: kim <grufwub@gmail.com> * fuck you linter Signed-off-by: kim <grufwub@gmail.com>
2022-05-15 11:16:43 +02:00
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package httpclient_test
import (
"bytes"
"errors"
"io"
"net/http"
"net/http/httptest"
"net/netip"
"testing"
"github.com/superseriousbusiness/gotosocial/internal/httpclient"
)
var privateIPs = []string{
"http://127.0.0.1:80",
"http://0.0.0.0:80",
"http://192.168.0.1:80",
"http://192.168.1.0:80",
"http://10.0.0.0:80",
"http://172.16.0.0:80",
"http://10.255.255.255:80",
"http://172.31.255.255:80",
"http://255.255.255.255:80",
}
var bodies = []string{
"hello world!",
"{}",
`{"key": "value", "some": "kinda bullshit"}`,
"body with\r\nnewlines",
}
// Note:
// There is no test for the .MaxOpenConns implementation
// in the httpclient.Client{}, due to the difficult to test
// this. The block is only held for the actual dial out to
// the connection, so the usual test of blocking and holding
// open this queue slot to check we can't open another isn't
// an easy test here.
func TestHTTPClientSmallBody(t *testing.T) {
for _, body := range bodies {
_TestHTTPClientWithBody(t, []byte(body), int(^uint16(0)))
}
}
func TestHTTPClientExactBody(t *testing.T) {
for _, body := range bodies {
_TestHTTPClientWithBody(t, []byte(body), len(body))
}
}
func TestHTTPClientLargeBody(t *testing.T) {
for _, body := range bodies {
_TestHTTPClientWithBody(t, []byte(body), len(body)-1)
}
}
func _TestHTTPClientWithBody(t *testing.T, body []byte, max int) {
var (
handler http.HandlerFunc
expect []byte
expectErr error
)
// If this is a larger body, reslice and
// set error so we know what to expect
expect = body
if max < len(body) {
expect = expect[:max]
expectErr = httpclient.ErrBodyTooLarge
}
// Create new HTTP client with maximum body size
client := httpclient.New(httpclient.Config{
MaxBodySize: int64(max),
DisableCompression: true,
AllowRanges: []netip.Prefix{
// Loopback (used by server)
netip.MustParsePrefix("127.0.0.1/8"),
},
})
// Set simple body-writing test handler
handler = func(rw http.ResponseWriter, r *http.Request) {
_, _ = rw.Write(body)
}
// Start the test server
srv := httptest.NewServer(handler)
defer srv.Close()
// Wrap body to provide reader iface
rbody := bytes.NewReader(body)
// Create the test HTTP request
req, _ := http.NewRequest("POST", srv.URL, rbody)
// Perform the test request
rsp, err := client.Do(req)
if !errors.Is(err, expectErr) {
t.Fatalf("error performing client request: %v", err)
} else if err != nil {
return // expected error
}
defer rsp.Body.Close()
// Read response body into memory
check, err := io.ReadAll(rsp.Body)
if err != nil {
t.Fatalf("error reading response body: %v", err)
}
// Check actual response body matches expected
if !bytes.Equal(expect, check) {
t.Errorf("response body did not match expected: expect=%q actual=%q", string(expect), string(check))
}
}
func TestHTTPClientPrivateIP(t *testing.T) {
client := httpclient.New(httpclient.Config{})
for _, addr := range privateIPs {
// Prepare request to private IP
req, _ := http.NewRequest("GET", addr, nil)
// Perform the HTTP request
_, err := client.Do(req)
if !errors.Is(err, httpclient.ErrReservedAddr) {
t.Errorf("dialing private address did not return expected error: %v", err)
}
}
}