mirror of
https://github.com/superseriousbusiness/gotosocial
synced 2025-06-05 21:59:39 +02:00
[security] transport.Controller{} and transport.Transport{} security and performance improvements (#564)
* cache transports in controller by privkey-generated pubkey, add retry logic to transport requests Signed-off-by: kim <grufwub@gmail.com> * update code comments, defer mutex unlocks Signed-off-by: kim <grufwub@gmail.com> * add count to 'performing request' log message Signed-off-by: kim <grufwub@gmail.com> * reduce repeated conversions of same url.URL object Signed-off-by: kim <grufwub@gmail.com> * move worker.Worker to concurrency subpackage, add WorkQueue type, limit transport http client use by WorkQueue Signed-off-by: kim <grufwub@gmail.com> * fix security advisories regarding max outgoing conns, max rsp body size - implemented by a new httpclient.Client{} that wraps an underlying client with a queue to limit connections, and limit reader wrapping a response body with a configured maximum size - update pub.HttpClient args passed around to be this new httpclient.Client{} Signed-off-by: kim <grufwub@gmail.com> * add httpclient tests, move ip validation to separate package + change mechanism Signed-off-by: kim <grufwub@gmail.com> * fix merge conflicts Signed-off-by: kim <grufwub@gmail.com> * use singular mutex in transport rather than separate signer mus Signed-off-by: kim <grufwub@gmail.com> * improved useragent string Signed-off-by: kim <grufwub@gmail.com> * add note regarding missing test Signed-off-by: kim <grufwub@gmail.com> * remove useragent field from transport (instead store in controller) Signed-off-by: kim <grufwub@gmail.com> * shutup linter Signed-off-by: kim <grufwub@gmail.com> * reset other signing headers on each loop iteration Signed-off-by: kim <grufwub@gmail.com> * respect request ctx during retry-backoff sleep period Signed-off-by: kim <grufwub@gmail.com> * use external pkg with docs explaining performance "hack" Signed-off-by: kim <grufwub@gmail.com> * use http package constants instead of string method literals Signed-off-by: kim <grufwub@gmail.com> * add license file headers Signed-off-by: kim <grufwub@gmail.com> * update code comment to match new func names Signed-off-by: kim <grufwub@gmail.com> * updates to user-agent string Signed-off-by: kim <grufwub@gmail.com> * update signed testrig models to fit with new transport logic (instead uses separate signer now) Signed-off-by: kim <grufwub@gmail.com> * fuck you linter Signed-off-by: kim <grufwub@gmail.com>
This commit is contained in:
kim
GitHub
154
internal/httpclient/client_test.go
Normal file
154
internal/httpclient/client_test.go
Normal file
@@ -0,0 +1,154 @@
|
||||
/*
|
||||
GoToSocial
|
||||
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package httpclient_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/netip"
|
||||
"testing"
|
||||
|
||||
"github.com/superseriousbusiness/gotosocial/internal/httpclient"
|
||||
)
|
||||
|
||||
var privateIPs = []string{
|
||||
"http://127.0.0.1:80",
|
||||
"http://0.0.0.0:80",
|
||||
"http://192.168.0.1:80",
|
||||
"http://192.168.1.0:80",
|
||||
"http://10.0.0.0:80",
|
||||
"http://172.16.0.0:80",
|
||||
"http://10.255.255.255:80",
|
||||
"http://172.31.255.255:80",
|
||||
"http://255.255.255.255:80",
|
||||
}
|
||||
|
||||
var bodies = []string{
|
||||
"hello world!",
|
||||
"{}",
|
||||
`{"key": "value", "some": "kinda bullshit"}`,
|
||||
"body with\r\nnewlines",
|
||||
}
|
||||
|
||||
// Note:
|
||||
// There is no test for the .MaxOpenConns implementation
|
||||
// in the httpclient.Client{}, due to the difficult to test
|
||||
// this. The block is only held for the actual dial out to
|
||||
// the connection, so the usual test of blocking and holding
|
||||
// open this queue slot to check we can't open another isn't
|
||||
// an easy test here.
|
||||
|
||||
func TestHTTPClientSmallBody(t *testing.T) {
|
||||
for _, body := range bodies {
|
||||
_TestHTTPClientWithBody(t, []byte(body), int(^uint16(0)))
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPClientExactBody(t *testing.T) {
|
||||
for _, body := range bodies {
|
||||
_TestHTTPClientWithBody(t, []byte(body), len(body))
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPClientLargeBody(t *testing.T) {
|
||||
for _, body := range bodies {
|
||||
_TestHTTPClientWithBody(t, []byte(body), len(body)-1)
|
||||
}
|
||||
}
|
||||
|
||||
func _TestHTTPClientWithBody(t *testing.T, body []byte, max int) {
|
||||
var (
|
||||
handler http.HandlerFunc
|
||||
|
||||
expect []byte
|
||||
|
||||
expectErr error
|
||||
)
|
||||
|
||||
// If this is a larger body, reslice and
|
||||
// set error so we know what to expect
|
||||
expect = body
|
||||
if max < len(body) {
|
||||
expect = expect[:max]
|
||||
expectErr = httpclient.ErrBodyTooLarge
|
||||
}
|
||||
|
||||
// Create new HTTP client with maximum body size
|
||||
client := httpclient.New(httpclient.Config{
|
||||
MaxBodySize: int64(max),
|
||||
DisableCompression: true,
|
||||
AllowRanges: []netip.Prefix{
|
||||
// Loopback (used by server)
|
||||
netip.MustParsePrefix("127.0.0.1/8"),
|
||||
},
|
||||
})
|
||||
|
||||
// Set simple body-writing test handler
|
||||
handler = func(rw http.ResponseWriter, r *http.Request) {
|
||||
_, _ = rw.Write(body)
|
||||
}
|
||||
|
||||
// Start the test server
|
||||
srv := httptest.NewServer(handler)
|
||||
defer srv.Close()
|
||||
|
||||
// Wrap body to provide reader iface
|
||||
rbody := bytes.NewReader(body)
|
||||
|
||||
// Create the test HTTP request
|
||||
req, _ := http.NewRequest("POST", srv.URL, rbody)
|
||||
|
||||
// Perform the test request
|
||||
rsp, err := client.Do(req)
|
||||
if !errors.Is(err, expectErr) {
|
||||
t.Fatalf("error performing client request: %v", err)
|
||||
} else if err != nil {
|
||||
return // expected error
|
||||
}
|
||||
defer rsp.Body.Close()
|
||||
|
||||
// Read response body into memory
|
||||
check, err := io.ReadAll(rsp.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("error reading response body: %v", err)
|
||||
}
|
||||
|
||||
// Check actual response body matches expected
|
||||
if !bytes.Equal(expect, check) {
|
||||
t.Errorf("response body did not match expected: expect=%q actual=%q", string(expect), string(check))
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPClientPrivateIP(t *testing.T) {
|
||||
client := httpclient.New(httpclient.Config{})
|
||||
|
||||
for _, addr := range privateIPs {
|
||||
// Prepare request to private IP
|
||||
req, _ := http.NewRequest("GET", addr, nil)
|
||||
|
||||
// Perform the HTTP request
|
||||
_, err := client.Do(req)
|
||||
if !errors.Is(err, httpclient.ErrReservedAddr) {
|
||||
t.Errorf("dialing private address did not return expected error: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user