[bugfix] 2643 bug search for account url doesnt always work when redirected (#2673)

* update activity library so dereferencer returns full response and checks *final* link to allow for redirects

* temporarily add bodged fixed library

* remove unused code

* update getAccountFeatured() to use dereferenceCollectionPage()

* make sure to release map

* perform a 2nd decode to ensure reader is empty after primary decode

* add comment explaining choice of using Decode() instead of Unmarshal()

* update embedded activity library to latest matching https://github.com/superseriousbusiness/activity/pull/21

* add checks to look for changed URI and re-check database if redirected

* update max iteration count to 512, add checks during dereferenceAncestors() for indirect URLs

* remove doubled-up code

* fix use of status instead of current

* use URIs for checking equality for security

* use the latest known URI for boost_of_uri in case original was an indirect

* add dereferenceCollection() function for dereferenceAccountFeatured()

* pull in latest github.com/superseriousbusiness/activity version (and remove the bodge!!)

* fix typo in code comments

* update decodeType() to accept a readcloser and handle body closing

* switch to checking using BoostOfID and add note why not using BoostOfURI

* ensure InReplyTo gets unset when deleting status parent in case currently stubbed

* add tests for Collection and CollectionPage iterators

vendor/github.com/superseriousbusiness/activity/pub/base_actor.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 
@@ -68,12 +67,7 @@ func NewSocialActor(c CommonBehavior,
 	clock Clock) Actor {
 	return &baseActor{
 		// Use SideEffectActor without s2s.
-		delegate: &SideEffectActor{
-			common: c,
-			c2s:    c2s,
-			db:     db,
-			clock:  clock,
-		},
+		delegate:             NewSideEffectActor(c, nil, c2s, db, clock),
 		enableSocialProtocol: true,
 		clock:                clock,
 	}
@@ -98,12 +92,7 @@ func NewFederatingActor(c CommonBehavior,
 	return &baseActorFederating{
 		baseActor{
 			// Use SideEffectActor without c2s.
-			delegate: &SideEffectActor{
-				common: c,
-				s2s:    s2s,
-				db:     db,
-				clock:  clock,
-			},
+			delegate:                NewSideEffectActor(c, s2s, nil, db, clock),
 			enableFederatedProtocol: true,
 			clock:                   clock,
 		},
@@ -195,14 +184,10 @@ func (b *baseActor) PostInboxScheme(c context.Context, w http.ResponseWriter, r
 	// Begin processing the request, but have not yet applied
 	// authorization (ex: blocks). Obtain the activity reject unknown
 	// activities.
-	raw, err := ioutil.ReadAll(r.Body)
+	m, err := readActivityPubReq(r)
 	if err != nil {
 		return true, err
 	}
-	var m map[string]interface{}
-	if err = json.Unmarshal(raw, &m); err != nil {
-		return true, err
-	}
 	asValue, err := streams.ToType(c, m)
 	if err != nil && !streams.IsUnmatchedErr(err) {
 		return true, err
@@ -340,14 +325,10 @@ func (b *baseActor) PostOutboxScheme(c context.Context, w http.ResponseWriter, r
 		return true, nil
 	}
 	// Everything is good to begin processing the request.
-	raw, err := ioutil.ReadAll(r.Body)
+	m, err := readActivityPubReq(r)
 	if err != nil {
 		return true, err
 	}
-	var m map[string]interface{}
-	if err = json.Unmarshal(raw, &m); err != nil {
-		return true, err
-	}
 	// Note that converting to a Type will NOT successfully convert types
 	// not known to go-fed. This prevents accidentally wrapping an Activity
 	// type unknown to go-fed in a Create below. Instead,

vendor/github.com/superseriousbusiness/activity/pub/federating_wrapped_callbacks.go

@@ -2,7 +2,6 @@ package pub
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"net/url"
 
@@ -244,12 +243,12 @@ func (w FederatingWrappedCallbacks) create(c context.Context, a vocab.ActivitySt
 			if err != nil {
 				return err
 			}
-			b, err := tport.Dereference(c, iter.GetIRI())
+			resp, err := tport.Dereference(c, iter.GetIRI())
 			if err != nil {
 				return err
 			}
-			var m map[string]interface{}
-			if err = json.Unmarshal(b, &m); err != nil {
+			m, err := readActivityPubResp(resp)
+			if err != nil {
 				return err
 			}
 			t, err = streams.ToType(c, m)
@@ -514,12 +513,12 @@ func (w FederatingWrappedCallbacks) accept(c context.Context, a vocab.ActivitySt
 				if err != nil {
 					return err
 				}
-				b, err := tport.Dereference(c, iter.GetIRI())
+				resp, err := tport.Dereference(c, iter.GetIRI())
 				if err != nil {
 					return err
 				}
-				var m map[string]interface{}
-				if err = json.Unmarshal(b, &m); err != nil {
+				m, err := readActivityPubResp(resp)
+				if err != nil {
 					return err
 				}
 				t, err = streams.ToType(c, m)

vendor/github.com/superseriousbusiness/activity/pub/side_effect_actor.go

@@ -629,14 +629,14 @@ func (a *SideEffectActor) hasInboxForwardingValues(c context.Context, inboxIRI *
 		if err != nil {
 			return false, err
 		}
-		b, err := tport.Dereference(c, iri)
+		resp, err := tport.Dereference(c, iri)
 		if err != nil {
 			// Do not fail the entire process if the data is
 			// missing.
 			continue
 		}
-		var m map[string]interface{}
-		if err = json.Unmarshal(b, &m); err != nil {
+		m, err := readActivityPubResp(resp)
+		if err != nil {
 			return false, err
 		}
 		t, err := streams.ToType(c, m)
@@ -855,13 +855,14 @@ func (a *SideEffectActor) resolveActors(c context.Context, t Transport, r []*url
 // The returned actor could be nil, if it wasn't an actor (ex: a Collection or
 // OrderedCollection).
 func (a *SideEffectActor) dereferenceForResolvingInboxes(c context.Context, t Transport, actorIRI *url.URL) (actor vocab.Type, moreActorIRIs []*url.URL, err error) {
-	var resp []byte
+	var resp *http.Response
 	resp, err = t.Dereference(c, actorIRI)
 	if err != nil {
 		return
 	}
 	var m map[string]interface{}
-	if err = json.Unmarshal(resp, &m); err != nil {
+	m, err = readActivityPubResp(resp)
+	if err != nil {
 		return
 	}
 	actor, err = streams.ToType(c, m)

vendor/github.com/superseriousbusiness/activity/pub/transport.go

@@ -5,7 +5,6 @@ import (
 	"context"
 	"crypto"
 	"fmt"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 	"strings"
@@ -40,11 +39,13 @@ func isSuccess(code int) bool {
 //
 // It may be reused multiple times, but never concurrently.
 type Transport interface {
-	// Dereference fetches the ActivityStreams object located at this IRI
-	// with a GET request.
-	Dereference(c context.Context, iri *url.URL) ([]byte, error)
+	// Dereference fetches the ActivityStreams object located at this IRI with
+	// a GET request. Note that Response will only be returned on status = OK.
+	Dereference(c context.Context, iri *url.URL) (*http.Response, error)
+
 	// Deliver sends an ActivityStreams object.
 	Deliver(c context.Context, b []byte, to *url.URL) error
+
 	// BatchDeliver sends an ActivityStreams object to multiple recipients.
 	BatchDeliver(c context.Context, b []byte, recipients []*url.URL) error
 }
@@ -107,9 +108,8 @@ func NewHttpSigTransport(
 	}
 }
 
-// Dereference sends a GET request signed with an HTTP Signature to obtain an
-// ActivityStreams value.
-func (h HttpSigTransport) Dereference(c context.Context, iri *url.URL) ([]byte, error) {
+// Dereference sends a GET request signed with an HTTP Signature to obtain an ActivityStreams value.
+func (h HttpSigTransport) Dereference(c context.Context, iri *url.URL) (*http.Response, error) {
 	req, err := http.NewRequest("GET", iri.String(), nil)
 	if err != nil {
 		return nil, err
@@ -130,11 +130,11 @@ func (h HttpSigTransport) Dereference(c context.Context, iri *url.URL) ([]byte,
 	if err != nil {
 		return nil, err
 	}
-	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK {
+		_ = resp.Body.Close()
 		return nil, fmt.Errorf("GET request to %s failed (%d): %s", iri.String(), resp.StatusCode, resp.Status)
 	}
-	return ioutil.ReadAll(resp.Body)
+	return resp, nil
 }
 
 // Deliver sends a POST request with an HTTP Signature.
@@ -166,8 +166,7 @@ func (h HttpSigTransport) Deliver(c context.Context, b []byte, to *url.URL) erro
 	return nil
 }
 
-// BatchDeliver sends concurrent POST requests. Returns an error if any of the
-// requests had an error.
+// BatchDeliver sends concurrent POST requests. Returns an error if any of the requests had an error.
 func (h HttpSigTransport) BatchDeliver(c context.Context, b []byte, recipients []*url.URL) error {
 	var wg sync.WaitGroup
 	errCh := make(chan error, len(recipients))

vendor/github.com/superseriousbusiness/activity/pub/util.go

@@ -8,6 +8,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"net/http"
 	"net/url"
 	"strings"
@@ -71,6 +72,59 @@ const (
 	acceptHeader = "Accept"
 )
 
+// readActivityPubReq reads ActivityPub data from an incoming request, handling body close.
+func readActivityPubReq(req *http.Request) (map[string]interface{}, error) {
+	// Ensure closed when done.
+	defer req.Body.Close()
+
+	var m map[string]interface{}
+
+	// Wrap body in a JSON decoder.
+	dec := json.NewDecoder(req.Body)
+
+	// Decode JSON body as "raw" AP data map.
+	if err := dec.Decode(&m); err != nil {
+		return nil, err
+	}
+
+	// Perform a final second decode to ensure no trailing
+	// garbage data or second JSON value (indicates malformed).
+	if err := dec.Decode(&struct{}{}); err != io.EOF {
+		return nil, errors.New("trailing data after json")
+	}
+
+	return m, nil
+}
+
+// readActivityPubResp reads ActivityPub data from a dereference response, handling media type check and body close.
+func readActivityPubResp(resp *http.Response) (map[string]interface{}, error) {
+	// Ensure closed when done.
+	defer resp.Body.Close()
+
+	// Check the incoming response media type is the expected ActivityPub content-type.
+	if mediaType := resp.Header.Get("Content-Type"); !headerIsActivityPubMediaType(mediaType) {
+		return nil, fmt.Errorf("%s %s resp was not ActivityPub media type: %s", resp.Request.Method, resp.Request.URL, mediaType)
+	}
+
+	var m map[string]interface{}
+
+	// Wrap body in a JSON decoder.
+	dec := json.NewDecoder(resp.Body)
+
+	// Decode JSON body as "raw" AP data map.
+	if err := dec.Decode(&m); err != nil {
+		return nil, err
+	}
+
+	// Perform a final second decode to ensure no trailing
+	// garbage data or second JSON value (indicates malformed).
+	if err := dec.Decode(&struct{}{}); err != io.EOF {
+		return nil, errors.New("trailing data after json")
+	}
+
+	return m, nil
+}
+
 // isActivityPubPost returns true if the request is a POST request that has the
 // ActivityStreams content type header
 func isActivityPubPost(r *http.Request) bool {
@@ -774,12 +828,12 @@ func mustHaveActivityActorsMatchObjectActors(c context.Context,
 		if err != nil {
 			return err
 		}
-		b, err := tport.Dereference(c, iri)
+		resp, err := tport.Dereference(c, iri)
 		if err != nil {
 			return err
 		}
-		var m map[string]interface{}
-		if err = json.Unmarshal(b, &m); err != nil {
+		m, err := readActivityPubResp(resp)
+		if err != nil {
 			return err
 		}
 		t, err := streams.ToType(c, m)