[feature/internal/httpclient] add option to configure outgoing protocol (#4134)

# Description

Adds a configuration option to set the outgoing transport of HTTP requests for testing purposes.

## Checklist

- [x] I/we have read the [GoToSocial contribution guidelines](https://codeberg.org/superseriousbusiness/gotosocial/src/branch/main/CONTRIBUTING.md).
- [ ] I/we have discussed the proposed changes already, either in an issue on the repository, or in the Matrix chat.
- [x] I/we have not leveraged AI to create the proposed changes.
- [x] I/we have performed a self-review of added code.
- [x] I/we have written code that is legible and maintainable by others.
- [ ] I/we have commented the added code, particularly in hard-to-understand areas.
- [x] I/we have made any necessary changes to documentation.
- [ ] I/we have added tests that cover new code.
- [ ] I/we have run tests and they pass locally with the changes.
- [ ] I/we have run `go fmt ./...` and `golangci-lint run`.
  - Note: `go fmt` yes, golangci-lint fails, can't load any configuration

Reviewed-on: https://codeberg.org/superseriousbusiness/gotosocial/pulls/4134
Reviewed-by: kim <gruf@noreply.codeberg.org>
Co-authored-by: famfo <famfo@famfo.xyz>
Co-committed-by: famfo <famfo@famfo.xyz>

docs/configuration/httpclient.md

@@ -60,9 +60,18 @@ http-client:
   #
   # THIS SETTING SHOULD BE USED FOR TESTING ONLY! IF YOU TURN THIS
   # ON WHILE RUNNING IN PRODUCTION YOU ARE LEAVING YOUR SERVER WIDE
-  # OPEN TO MAN IN THE MIDDLE ATTACKS! DO NOT CHANGE THIS SETTING 
+  # OPEN TO MAN IN THE MIDDLE ATTACKS! DO NOT CHANGE THIS SETTING
   # UNLESS YOU KNOW EXACTLY WHAT YOU'RE DOING AND WHY YOU'RE DOING IT.
   #
   # Default: false
   tls-insecure-skip-verify: false
+
+  # Bool. Sets outgoing queries to webfinger, host-meta and nodeinfo to use
+  # HTTP instead of HTTPS.
+  #
+  # THIS SETTING SHOULD BE USED FOR TESTING ONLY! DO NOT CHANGE THIS SETTING
+  # UNLESS YOU KNOW EXACTLY WHAT YOU'RE DOING AND WHY YOU'RE DOING IT.
+  #
+  # Default: false
+  insecure-outgoing: false
 ```

example/config.yaml

@@ -1113,6 +1113,15 @@ http-client:
   # Default: false
   tls-insecure-skip-verify: false
 
+  # Bool. Sets outgoing queries to webfinger, host-meta and nodeinfo to use
+  # HTTP instead of HTTPS.
+  #
+  # THIS SETTING SHOULD BE USED FOR TESTING ONLY! DO NOT CHANGE THIS SETTING
+  # UNLESS YOU KNOW EXACTLY WHAT YOU'RE DOING AND WHY YOU'RE DOING IT.
+  #
+  # Default: false
+  insecure-outgoing: false
+
 #############################
 ##### ADVANCED SETTINGS #####
 #############################

internal/config/config.go

@@ -198,6 +198,7 @@ type HTTPClientConfiguration struct {
 	BlockIPs              []string      `name:"block-ips"`
 	Timeout               time.Duration `name:"timeout"`
 	TLSInsecureSkipVerify bool          `name:"tls-insecure-skip-verify"`
+	InsecureOutgoing      bool          `name:"insecure-outgoing"`
 }
 
 type CacheConfiguration struct {

internal/config/helpers.gen.go

@@ -150,6 +150,7 @@ func (cfg *Configuration) RegisterFlags(flags *pflag.FlagSet) {
 	flags.StringSlice("http-client-block-ips", cfg.HTTPClient.BlockIPs, "")
 	flags.Duration("http-client-timeout", cfg.HTTPClient.Timeout, "")
 	flags.Bool("http-client-tls-insecure-skip-verify", cfg.HTTPClient.TLSInsecureSkipVerify, "")
+	flags.Bool("http-client-insecure-outgoing", cfg.HTTPClient.InsecureOutgoing, "")
 	flags.String("cache-memory-target", cfg.Cache.MemoryTarget.String(), "")
 	flags.Float64("cache-account-mem-ratio", cfg.Cache.AccountMemRatio, "")
 	flags.Float64("cache-account-note-mem-ratio", cfg.Cache.AccountNoteMemRatio, "")
@@ -333,6 +334,7 @@ func (cfg *Configuration) MarshalMap() map[string]any {
 	cfgmap["http-client-block-ips"] = cfg.HTTPClient.BlockIPs
 	cfgmap["http-client-timeout"] = cfg.HTTPClient.Timeout
 	cfgmap["http-client-tls-insecure-skip-verify"] = cfg.HTTPClient.TLSInsecureSkipVerify
+	cfgmap["http-client-insecure-outgoing"] = cfg.HTTPClient.InsecureOutgoing
 	cfgmap["cache-memory-target"] = cfg.Cache.MemoryTarget.String()
 	cfgmap["cache-account-mem-ratio"] = cfg.Cache.AccountMemRatio
 	cfgmap["cache-account-note-mem-ratio"] = cfg.Cache.AccountNoteMemRatio
@@ -1406,6 +1408,14 @@ func (cfg *Configuration) UnmarshalMap(cfgmap map[string]any) error {
 		}
 	}
 
+	if ival, ok := cfgmap["http-client-insecure-outgoing"]; ok {
+		var err error
+		cfg.HTTPClient.InsecureOutgoing, err = cast.ToBoolE(ival)
+		if err != nil {
+			return fmt.Errorf("error casting %#v -> bool for 'http-client-insecure-outgoing': %w", ival, err)
+		}
+	}
+
 	if ival, ok := cfgmap["cache-memory-target"]; ok {
 		t, err := cast.ToStringE(ival)
 		if err != nil {
@@ -4969,6 +4979,31 @@ func GetHTTPClientTLSInsecureSkipVerify() bool { return global.GetHTTPClientTLSI
 // SetHTTPClientTLSInsecureSkipVerify safely sets the value for global configuration 'HTTPClient.TLSInsecureSkipVerify' field
 func SetHTTPClientTLSInsecureSkipVerify(v bool) { global.SetHTTPClientTLSInsecureSkipVerify(v) }
 
+// HTTPClientInsecureOutgoingFlag returns the flag name for the 'HTTPClient.InsecureOutgoing' field
+func HTTPClientInsecureOutgoingFlag() string { return "http-client-insecure-outgoing" }
+
+// GetHTTPClientInsecureOutgoing safely fetches the Configuration value for state's 'HTTPClient.InsecureOutgoing' field
+func (st *ConfigState) GetHTTPClientInsecureOutgoing() (v bool) {
+	st.mutex.RLock()
+	v = st.config.HTTPClient.InsecureOutgoing
+	st.mutex.RUnlock()
+	return
+}
+
+// SetHTTPClientInsecureOutgoing safely sets the Configuration value for state's 'HTTPClient.InsecureOutgoing' field
+func (st *ConfigState) SetHTTPClientInsecureOutgoing(v bool) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.HTTPClient.InsecureOutgoing = v
+	st.reloadToViper()
+}
+
+// GetHTTPClientInsecureOutgoing safely fetches the value for global configuration 'HTTPClient.InsecureOutgoing' field
+func GetHTTPClientInsecureOutgoing() bool { return global.GetHTTPClientInsecureOutgoing() }
+
+// SetHTTPClientInsecureOutgoing safely sets the value for global configuration 'HTTPClient.InsecureOutgoing' field
+func SetHTTPClientInsecureOutgoing(v bool) { global.SetHTTPClientInsecureOutgoing(v) }
+
 // CacheMemoryTargetFlag returns the flag name for the 'Cache.MemoryTarget' field
 func CacheMemoryTargetFlag() string { return "cache-memory-target" }
 
@@ -6849,6 +6884,17 @@ func flattenConfigMap(cfgmap map[string]any) {
 		}
 	}
 
+	for _, key := range [][]string{
+		{"http-client", "insecure-outgoing"},
+	} {
+		ival, ok := mapGet(cfgmap, key...)
+		if ok {
+			cfgmap["http-client-insecure-outgoing"] = ival
+			nestedKeys[key[0]] = struct{}{}
+			break
+		}
+	}
+
 	for _, key := range [][]string{
 		{"cache", "memory-target"},
 	} {

internal/config/types.go

@@ -48,3 +48,11 @@ func (p *IPPrefixes) Strings() []string {
 	}
 	return strs
 }
+
+func GetHTTPClientOutgoingScheme() (schema string) {
+	if GetHTTPClientInsecureOutgoing() {
+		return "http://"
+	}
+
+	return "https://"
+}

internal/config/validate.go

@@ -181,5 +181,13 @@ func Validate() error {
 		)
 	}
 
+	// http-client.insecure-outgoing
+	if GetHTTPClientInsecureOutgoing() {
+		log.Warn(nil, "http-client.insecure-outgoing was set to TRUE. "+
+			"*****THIS SHOULD BE USED FOR TESTING ONLY, IF YOU TURN THIS ON WHILE "+
+			"IF IN DOUBT, STOP YOUR SERVER *NOW* AND ADJUST YOUR CONFIGURATION!*****",
+		)
+	}
+
 	return errs.Combine()
 }

internal/transport/finger.go

@@ -27,6 +27,7 @@ import (
 
 	apimodel "code.superseriousbusiness.org/gotosocial/internal/api/model"
 	apiutil "code.superseriousbusiness.org/gotosocial/internal/api/util"
+	"code.superseriousbusiness.org/gotosocial/internal/config"
 	"code.superseriousbusiness.org/gotosocial/internal/gtserror"
 	"code.superseriousbusiness.org/gotosocial/internal/util"
 )
@@ -35,7 +36,7 @@ import (
 // well as if the URL was retrieved from cache. When the URL is retrieved
 // from cache we don't have to try and do host-meta discovery
 func (t *transport) webfingerURLFor(targetDomain string) (string, bool) {
-	url := "https://" + targetDomain + "/.well-known/webfinger"
+	url := config.GetHTTPClientOutgoingScheme() + targetDomain + "/.well-known/webfinger"
 
 	wc := t.controller.state.Caches.Webfinger
 
@@ -185,7 +186,7 @@ func (t *transport) Finger(ctx context.Context, targetUsername string, targetDom
 
 func (t *transport) webfingerFromHostMeta(ctx context.Context, targetDomain string) (string, error) {
 	// Build the request for the host-meta endpoint
-	hmurl := "https://" + targetDomain + "/.well-known/host-meta"
+	hmurl := config.GetHTTPClientOutgoingScheme() + targetDomain + "/.well-known/host-meta"
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, hmurl, nil)
 	if err != nil {
 		return "", err

test/envparsing.sh

@@ -105,6 +105,7 @@ EXPECT=$(cat << "EOF"
     "host": "example.com",
     "http-client-allow-ips": [],
     "http-client-block-ips": [],
+    "http-client-insecure-outgoing": false,
     "http-client-timeout": 30000000000,
     "http-client-tls-insecure-skip-verify": false,
     "instance-allow-backdating-statuses": true,