[bugfix] fix possible mutex lockup during streaming code (#2633)

* rewrite Stream{} to use much less mutex locking, update related code

* use new context for the stream context

* ensure stream gets closed on return of writeTo / readFrom WSConn()

* ensure stream write timeout gets cancelled

* remove embedded context type from Stream{}, reformat log messages for consistency

* use c.Request.Context() for context passed into Stream().Open()

* only return 1 boolean, fix tests to expect multiple stream types in messages

* changes to ping logic

* further improved ping logic

* don't export unused function types, update message sending to only include relevant stream type

* ensure stream gets closed 🤦

* update to error log on failed json marshal (instead of panic)

* inverse websocket read error checking to _ignore_ expected close errors

internal/processing/workers/fromclientapi_test.go

@@ -116,23 +116,20 @@ func (suite *FromClientAPITestSuite) checkStreamed(
 	expectPayload string,
 	expectEventType string,
 ) {
-	var msg *stream.Message
-streamLoop:
-	for {
-		select {
-		case msg = <-str.Messages:
-			break streamLoop // Got it.
-		case <-time.After(5 * time.Second):
-			break streamLoop // Didn't get it.
-		}
+
+	// Set a 5s timeout on context.
+	ctx := context.Background()
+	ctx, cncl := context.WithTimeout(ctx, time.Second*5)
+	defer cncl()
+
+	msg, ok := str.Recv(ctx)
+
+	if expectMessage && !ok {
+		suite.FailNow("expected a message but message was not received")
 	}
 
-	if expectMessage && msg == nil {
-		suite.FailNow("expected a message but message was nil")
-	}
-
-	if !expectMessage && msg != nil {
-		suite.FailNow("expected no message but message was not nil")
+	if !expectMessage && ok {
+		suite.FailNow("expected no message but message was received")
 	}
 
 	if expectPayload != "" && msg.Payload != expectPayload {