[feature] Allow users to skip http client tls verification for testing purposes (with appropriately loud warnings) (#2052)

This commit is contained in:
tobi 2023-08-01 19:50:17 +02:00 committed by GitHub
parent 9bd03e122e
commit 2be83fdca5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 98 additions and 16 deletions

View File

@ -437,17 +437,21 @@ Although this test *is* part of the CI/CD testing process, you probably won't ne
#### Federation #### Federation
By using the support for loading TLS files from disk it is possible to have two local instances with TLS to allow for (manually) testing federation. By using the support for loading TLS files from disk it is possible to have two or more local instances with TLS to allow for (manually) testing federation.
You'll need to set the following configuration options: You'll need to set the following configuration options:
* `GTS_TLS_CERTIFICATE_CHAIN`: poiting to a PEM-encoded certificate chain including the public certificate
* `GTS_TLS_CERTIFICATE_KEY`: pointing to a PEM-encoded private key - `GTS_TLS_CERTIFICATE_CHAIN`: poiting to a PEM-encoded certificate chain including the public certificate.
- `GTS_TLS_CERTIFICATE_KEY`: pointing to a PEM-encoded private key.
Additionally, for the Go HTTP client to recognise certificates issued by a custom CA as valid, you'll need to set one of: Additionally, for the Go HTTP client to recognise certificates issued by a custom CA as valid, you'll need to set one of:
* `SSL_CERT_FILE`: pointing to the public key of your custom CA
* `SSL_CERT_DIR`: a `:`-separated list of directories to load CA certificates from
You'll additionally need functioning DNS for your two instance names which you can achieve through entries in `/etc/hosts` or by running a local DNS server like [dnsmasq](https://thekelleys.org.uk/dnsmasq/doc.html). - `SSL_CERT_FILE`: pointing to the public key of your custom CA.
- `SSL_CERT_DIR`: a `:`-separated list of directories to load CA certificates from.
The above `SSL_CERT` variables work on Unix-like systems only, excluding Mac. See https://pkg.go.dev/crypto/x509#SystemCertPool. If you are running your tests on an architecture that doesn't support setting the above variables, you can instead disable TLS certificate verification for the HTTP client entirely by setting `http-client.tls-insecure-skip-verify` to `true` in the config.yaml file.
You'll additionally need functioning DNS for your two instance names, which you can achieve through entries in `/etc/hosts` or by running a local DNS server like [dnsmasq](https://thekelleys.org.uk/dnsmasq/doc.html).
### Updating Swagger docs ### Updating Swagger docs

View File

@ -107,9 +107,10 @@ var Start action.GTSAction = func(ctx context.Context) error {
// Build HTTP client // Build HTTP client
client := httpclient.New(httpclient.Config{ client := httpclient.New(httpclient.Config{
AllowRanges: config.MustParseIPPrefixes(config.GetHTTPClientAllowIPs()), AllowRanges: config.MustParseIPPrefixes(config.GetHTTPClientAllowIPs()),
BlockRanges: config.MustParseIPPrefixes(config.GetHTTPClientBlockIPs()), BlockRanges: config.MustParseIPPrefixes(config.GetHTTPClientBlockIPs()),
Timeout: config.GetHTTPClientTimeout(), Timeout: config.GetHTTPClientTimeout(),
TLSInsecureSkipVerify: config.GetHTTPClientTLSInsecureSkipVerify(),
}) })
// Initialize workers. // Initialize workers.

View File

@ -53,4 +53,16 @@ http-client:
# Both allow-ips and block-ips default to an empty array. # Both allow-ips and block-ips default to an empty array.
allow-ips: [] allow-ips: []
block-ips: [] block-ips: []
# Bool. Disable verification of TLS certificates of remote servers.
# With this set to 'true', GoToSocial will not error when a remote
# server presents an invalid or self-signed certificate.
#
# THIS SETTING SHOULD BE USED FOR TESTING ONLY! IF YOU TURN THIS
# ON WHILE RUNNING IN PRODUCTION YOU ARE LEAVING YOUR SERVER WIDE
# OPEN TO MAN IN THE MIDDLE ATTACKS! DO NOT CHANGE THIS SETTING
# UNLESS YOU KNOW EXACTLY WHAT YOU'RE DOING AND WHY YOU'RE DOING IT.
#
# Default: false
tls-insecure-skip-verify: false
``` ```

View File

@ -880,6 +880,18 @@ http-client:
allow-ips: [] allow-ips: []
block-ips: [] block-ips: []
# Bool. Disable verification of TLS certificates of remote servers.
# With this set to 'true', GoToSocial will not error when a remote
# server presents an invalid or self-signed certificate.
#
# THIS SETTING SHOULD BE USED FOR TESTING ONLY! IF YOU TURN THIS
# ON WHILE RUNNING IN PRODUCTION YOU ARE LEAVING YOUR SERVER WIDE
# OPEN TO MAN IN THE MIDDLE ATTACKS! DO NOT CHANGE THIS SETTING
# UNLESS YOU KNOW EXACTLY WHAT YOU'RE DOING AND WHY YOU'RE DOING IT.
#
# Default: false
tls-insecure-skip-verify: false
############################# #############################
##### ADVANCED SETTINGS ##### ##### ADVANCED SETTINGS #####
############################# #############################

View File

@ -168,9 +168,10 @@ type Configuration struct {
} }
type HTTPClientConfiguration struct { type HTTPClientConfiguration struct {
AllowIPs []string `name:"allow-ips"` AllowIPs []string `name:"allow-ips"`
BlockIPs []string `name:"block-ips"` BlockIPs []string `name:"block-ips"`
Timeout time.Duration `name:"timeout"` Timeout time.Duration `name:"timeout"`
TLSInsecureSkipVerify bool `name:"tls-insecure-skip-verify"`
} }
type CacheConfiguration struct { type CacheConfiguration struct {

View File

@ -234,9 +234,10 @@ var Defaults = Configuration{
}, },
HTTPClient: HTTPClientConfiguration{ HTTPClient: HTTPClientConfiguration{
AllowIPs: make([]string, 0), AllowIPs: make([]string, 0),
BlockIPs: make([]string, 0), BlockIPs: make([]string, 0),
Timeout: 10 * time.Second, Timeout: 10 * time.Second,
TLSInsecureSkipVerify: false,
}, },
AdminMediaPruneDryRun: true, AdminMediaPruneDryRun: true,

View File

@ -60,6 +60,7 @@ func (s *ConfigState) AddGlobalFlags(cmd *cobra.Command) {
cmd.PersistentFlags().StringSlice(HTTPClientAllowIPsFlag(), cfg.HTTPClient.AllowIPs, "no usage string") cmd.PersistentFlags().StringSlice(HTTPClientAllowIPsFlag(), cfg.HTTPClient.AllowIPs, "no usage string")
cmd.PersistentFlags().StringSlice(HTTPClientBlockIPsFlag(), cfg.HTTPClient.BlockIPs, "no usage string") cmd.PersistentFlags().StringSlice(HTTPClientBlockIPsFlag(), cfg.HTTPClient.BlockIPs, "no usage string")
cmd.PersistentFlags().Duration(HTTPClientTimeoutFlag(), cfg.HTTPClient.Timeout, "no usage string") cmd.PersistentFlags().Duration(HTTPClientTimeoutFlag(), cfg.HTTPClient.Timeout, "no usage string")
cmd.PersistentFlags().Bool(HTTPClientTLSInsecureSkipVerifyFlag(), cfg.HTTPClient.TLSInsecureSkipVerify, "no usage string")
}) })
} }

View File

@ -2399,6 +2399,31 @@ func GetHTTPClientTimeout() time.Duration { return global.GetHTTPClientTimeout()
// SetHTTPClientTimeout safely sets the value for global configuration 'HTTPClient.Timeout' field // SetHTTPClientTimeout safely sets the value for global configuration 'HTTPClient.Timeout' field
func SetHTTPClientTimeout(v time.Duration) { global.SetHTTPClientTimeout(v) } func SetHTTPClientTimeout(v time.Duration) { global.SetHTTPClientTimeout(v) }
// GetHTTPClientTLSInsecureSkipVerify safely fetches the Configuration value for state's 'HTTPClient.TLSInsecureSkipVerify' field
func (st *ConfigState) GetHTTPClientTLSInsecureSkipVerify() (v bool) {
st.mutex.RLock()
v = st.config.HTTPClient.TLSInsecureSkipVerify
st.mutex.RUnlock()
return
}
// SetHTTPClientTLSInsecureSkipVerify safely sets the Configuration value for state's 'HTTPClient.TLSInsecureSkipVerify' field
func (st *ConfigState) SetHTTPClientTLSInsecureSkipVerify(v bool) {
st.mutex.Lock()
defer st.mutex.Unlock()
st.config.HTTPClient.TLSInsecureSkipVerify = v
st.reloadToViper()
}
// HTTPClientTLSInsecureSkipVerifyFlag returns the flag name for the 'HTTPClient.TLSInsecureSkipVerify' field
func HTTPClientTLSInsecureSkipVerifyFlag() string { return "httpclient-tls-insecure-skip-verify" }
// GetHTTPClientTLSInsecureSkipVerify safely fetches the value for global configuration 'HTTPClient.TLSInsecureSkipVerify' field
func GetHTTPClientTLSInsecureSkipVerify() bool { return global.GetHTTPClientTLSInsecureSkipVerify() }
// SetHTTPClientTLSInsecureSkipVerify safely sets the value for global configuration 'HTTPClient.TLSInsecureSkipVerify' field
func SetHTTPClientTLSInsecureSkipVerify(v bool) { global.SetHTTPClientTLSInsecureSkipVerify(v) }
// GetCacheGTSAccountMaxSize safely fetches the Configuration value for state's 'Cache.GTS.AccountMaxSize' field // GetCacheGTSAccountMaxSize safely fetches the Configuration value for state's 'Cache.GTS.AccountMaxSize' field
func (st *ConfigState) GetCacheGTSAccountMaxSize() (v int) { func (st *ConfigState) GetCacheGTSAccountMaxSize() (v int) {
st.mutex.RLock() st.mutex.RLock()

View File

@ -19,6 +19,7 @@ package httpclient
import ( import (
"context" "context"
"crypto/tls"
"errors" "errors"
"fmt" "fmt"
"io" "io"
@ -86,6 +87,14 @@ type Config struct {
// BlockRanges blocks outgoing communiciations to given IP nets. // BlockRanges blocks outgoing communiciations to given IP nets.
BlockRanges []netip.Prefix BlockRanges []netip.Prefix
// TLSInsecureSkipVerify can be set to true to
// skip validation of remote TLS certificates.
//
// THIS SHOULD BE USED FOR TESTING ONLY, IF YOU
// TURN THIS ON WHILE RUNNING IN PRODUCTION YOU
// ARE LEAVING YOUR SERVER WIDE OPEN TO ATTACKS!
TLSInsecureSkipVerify bool
} }
// Client wraps an underlying http.Client{} to provide the following: // Client wraps an underlying http.Client{} to provide the following:
@ -139,11 +148,26 @@ func New(cfg Config) *Client {
c.client.Timeout = cfg.Timeout c.client.Timeout = cfg.Timeout
c.bodyMax = cfg.MaxBodySize c.bodyMax = cfg.MaxBodySize
// Prepare TLS config for transport.
tlsClientConfig := &tls.Config{
InsecureSkipVerify: cfg.TLSInsecureSkipVerify, //nolint:gosec
}
if tlsClientConfig.InsecureSkipVerify {
// Warn against playing silly buggers.
log.Warn(nil, "http-client.tls-insecure-skip-verify was set to TRUE. "+
"*****THIS SHOULD BE USED FOR TESTING ONLY, IF YOU TURN THIS ON WHILE "+
"RUNNING IN PRODUCTION YOU ARE LEAVING YOUR SERVER WIDE OPEN TO ATTACKS! "+
"IF IN DOUBT, STOP YOUR SERVER *NOW* AND ADJUST YOUR CONFIGURATION!*****",
)
}
// Set underlying HTTP client roundtripper. // Set underlying HTTP client roundtripper.
c.client.Transport = &http.Transport{ c.client.Transport = &http.Transport{
Proxy: http.ProxyFromEnvironment, Proxy: http.ProxyFromEnvironment,
ForceAttemptHTTP2: true, ForceAttemptHTTP2: true,
DialContext: d.DialContext, DialContext: d.DialContext,
TLSClientConfig: tlsClientConfig,
MaxIdleConns: cfg.MaxIdleConns, MaxIdleConns: cfg.MaxIdleConns,
IdleConnTimeout: 90 * time.Second, IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second, TLSHandshakeTimeout: 10 * time.Second,

View File

@ -119,7 +119,8 @@ EXPECT=$(cat <<"EOF"
"http-client": { "http-client": {
"allow-ips": [], "allow-ips": [],
"block-ips": [], "block-ips": [],
"timeout": 10000000000 "timeout": 10000000000,
"tls-insecure-skip-verify": false
}, },
"instance-deliver-to-shared-inboxes": false, "instance-deliver-to-shared-inboxes": false,
"instance-expose-peers": true, "instance-expose-peers": true,