Inbox post (#22)

Inbox POST from federated servers now working for statuses and follow requests. Follow request client API added. Start work on federating outgoing messages. Other fixes and changes/tidying up.
2025-06-05 21:59:39 +02:00 · 2021-05-15 11:58:11 +02:00
parent 742f985d5b
commit cc48294c31
58 changed files with 2248 additions and 366 deletions
--- a/internal/typeutils/accountable.go
+++ b/internal/typeutils/accountable.go
@@ -1,101 +0,0 @@
-/*
-   GoToSocial
-   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
-
-   This program is free software: you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published by
-   the Free Software Foundation, either version 3 of the License, or
-   (at your option) any later version.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-package typeutils
-
-import "github.com/go-fed/activity/streams/vocab"
-
-// Accountable represents the minimum activitypub interface for representing an 'account'.
-// This interface is fulfilled by: Person, Application, Organization, Service, and Group
-type Accountable interface {
-	withJSONLDId
-	withGetTypeName
-	withPreferredUsername
-	withIcon
-	withDisplayName
-	withImage
-	withSummary
-	withDiscoverable
-	withURL
-	withPublicKey
-	withInbox
-	withOutbox
-	withFollowing
-	withFollowers
-	withFeatured
-}
-
-type withJSONLDId interface {
-	GetJSONLDId() vocab.JSONLDIdProperty
-}
-
-type withGetTypeName interface {
-	GetTypeName() string
-}
-
-type withPreferredUsername interface {
-	GetActivityStreamsPreferredUsername() vocab.ActivityStreamsPreferredUsernameProperty
-}
-
-type withIcon interface {
-	GetActivityStreamsIcon() vocab.ActivityStreamsIconProperty
-}
-
-type withDisplayName interface {
-	GetActivityStreamsName() vocab.ActivityStreamsNameProperty
-}
-
-type withImage interface {
-	GetActivityStreamsImage() vocab.ActivityStreamsImageProperty
-}
-
-type withSummary interface {
-	GetActivityStreamsSummary() vocab.ActivityStreamsSummaryProperty
-}
-
-type withDiscoverable interface {
-	GetTootDiscoverable() vocab.TootDiscoverableProperty
-}
-
-type withURL interface {
-	GetActivityStreamsUrl() vocab.ActivityStreamsUrlProperty
-}
-
-type withPublicKey interface {
-	GetW3IDSecurityV1PublicKey() vocab.W3IDSecurityV1PublicKeyProperty
-}
-
-type withInbox interface {
-	GetActivityStreamsInbox() vocab.ActivityStreamsInboxProperty
-}
-
-type withOutbox interface {
-	GetActivityStreamsOutbox() vocab.ActivityStreamsOutboxProperty
-}
-
-type withFollowing interface {
-	GetActivityStreamsFollowing() vocab.ActivityStreamsFollowingProperty
-}
-
-type withFollowers interface {
-	GetActivityStreamsFollowers() vocab.ActivityStreamsFollowersProperty
-}
-
-type withFeatured interface {
-	GetTootFeatured() vocab.TootFeaturedProperty
-}
--- a/internal/typeutils/asextractionutil.go
+++ b/internal/typeutils/asextractionutil.go
@@ -25,8 +25,12 @@ import (
 	"errors"
 	"fmt"
 	"net/url"
+	"strings"
+	"time"

 	"github.com/go-fed/activity/pub"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
 )

 func extractPreferredUsername(i withPreferredUsername) (string, error) {
@@ -40,22 +44,89 @@ func extractPreferredUsername(i withPreferredUsername) (string, error) {
 	return u.GetXMLSchemaString(), nil
 }

-func extractName(i withDisplayName) (string, error) {
+func extractName(i withName) (string, error) {
 	nameProp := i.GetActivityStreamsName()
 	if nameProp == nil {
 		return "", errors.New("activityStreamsName not found")
 	}

 	// take the first name string we can find
-	for nameIter := nameProp.Begin(); nameIter != nameProp.End(); nameIter = nameIter.Next() {
-		if nameIter.IsXMLSchemaString() && nameIter.GetXMLSchemaString() != "" {
-			return nameIter.GetXMLSchemaString(), nil
+	for iter := nameProp.Begin(); iter != nameProp.End(); iter = iter.Next() {
+		if iter.IsXMLSchemaString() && iter.GetXMLSchemaString() != "" {
+			return iter.GetXMLSchemaString(), nil
 		}
 	}

 	return "", errors.New("activityStreamsName not found")
 }

+func extractInReplyToURI(i withInReplyTo) (*url.URL, error) {
+	inReplyToProp := i.GetActivityStreamsInReplyTo()
+	for iter := inReplyToProp.Begin(); iter != inReplyToProp.End(); iter = iter.Next() {
+		if iter.IsIRI() {
+			if iter.GetIRI() != nil {
+				return iter.GetIRI(), nil
+			}
+		}
+	}
+	return nil, errors.New("couldn't find iri for in reply to")
+}
+
+func extractTos(i withTo) ([]*url.URL, error) {
+	to := []*url.URL{}
+	toProp := i.GetActivityStreamsTo()
+	for iter := toProp.Begin(); iter != toProp.End(); iter = iter.Next() {
+		if iter.IsIRI() {
+			if iter.GetIRI() != nil {
+				to = append(to, iter.GetIRI())
+			}
+		}
+	}
+	return to, nil
+}
+
+func extractCCs(i withCC) ([]*url.URL, error) {
+	cc := []*url.URL{}
+	ccProp := i.GetActivityStreamsCc()
+	for iter := ccProp.Begin(); iter != ccProp.End(); iter = iter.Next() {
+		if iter.IsIRI() {
+			if iter.GetIRI() != nil {
+				cc = append(cc, iter.GetIRI())
+			}
+		}
+	}
+	return cc, nil
+}
+
+func extractAttributedTo(i withAttributedTo) (*url.URL, error) {
+	attributedToProp := i.GetActivityStreamsAttributedTo()
+	for iter := attributedToProp.Begin(); iter != attributedToProp.End(); iter = iter.Next() {
+		if iter.IsIRI() {
+			if iter.GetIRI() != nil {
+				return iter.GetIRI(), nil
+			}
+		}
+	}
+	return nil, errors.New("couldn't find iri for attributed to")
+}
+
+func extractPublished(i withPublished) (time.Time, error) {
+	publishedProp := i.GetActivityStreamsPublished()
+	if publishedProp == nil {
+		return time.Time{}, errors.New("published prop was nil")
+	}
+
+	if !publishedProp.IsXMLSchemaDateTime() {
+		return time.Time{}, errors.New("published prop was not date time")
+	}
+
+	t := publishedProp.Get()
+	if t.IsZero() {
+		return time.Time{}, errors.New("published time was zero")
+	}
+	return t, nil
+}
+
 // extractIconURL extracts a URL to a supported image file from something like:
 //   "icon": {
 //     "mediaType": "image/jpeg",
@@ -72,12 +143,12 @@ func extractIconURL(i withIcon) (*url.URL, error) {
 	// here in order to find the first one that meets these criteria:
 	// 1. is an image
 	// 2. has a URL so we can grab it
-	for iconIter := iconProp.Begin(); iconIter != iconProp.End(); iconIter = iconIter.Next() {
+	for iter := iconProp.Begin(); iter != iconProp.End(); iter = iter.Next() {
 		// 1. is an image
-		if !iconIter.IsActivityStreamsImage() {
+		if !iter.IsActivityStreamsImage() {
 			continue
 		}
-		imageValue := iconIter.GetActivityStreamsImage()
+		imageValue := iter.GetActivityStreamsImage()
 		if imageValue == nil {
 			continue
 		}
@@ -108,12 +179,12 @@ func extractImageURL(i withImage) (*url.URL, error) {
 	// here in order to find the first one that meets these criteria:
 	// 1. is an image
 	// 2. has a URL so we can grab it
-	for imageIter := imageProp.Begin(); imageIter != imageProp.End(); imageIter = imageIter.Next() {
+	for iter := imageProp.Begin(); iter != imageProp.End(); iter = iter.Next() {
 		// 1. is an image
-		if !imageIter.IsActivityStreamsImage() {
+		if !iter.IsActivityStreamsImage() {
 			continue
 		}
-		imageValue := imageIter.GetActivityStreamsImage()
+		imageValue := iter.GetActivityStreamsImage()
 		if imageValue == nil {
 			continue
 		}
@@ -134,9 +205,9 @@ func extractSummary(i withSummary) (string, error) {
 		return "", errors.New("summary property was nil")
 	}

-	for summaryIter := summaryProp.Begin(); summaryIter != summaryProp.End(); summaryIter = summaryIter.Next() {
-		if summaryIter.IsXMLSchemaString() && summaryIter.GetXMLSchemaString() != "" {
-			return summaryIter.GetXMLSchemaString(), nil
+	for iter := summaryProp.Begin(); iter != summaryProp.End(); iter = iter.Next() {
+		if iter.IsXMLSchemaString() && iter.GetXMLSchemaString() != "" {
+			return iter.GetXMLSchemaString(), nil
 		}
 	}

@@ -156,9 +227,9 @@ func extractURL(i withURL) (*url.URL, error) {
 		return nil, errors.New("url property was nil")
 	}

-	for urlIter := urlProp.Begin(); urlIter != urlProp.End(); urlIter = urlIter.Next() {
-		if urlIter.IsIRI() && urlIter.GetIRI() != nil {
-			return urlIter.GetIRI(), nil
+	for iter := urlProp.Begin(); iter != urlProp.End(); iter = iter.Next() {
+		if iter.IsIRI() && iter.GetIRI() != nil {
+			return iter.GetIRI(), nil
 		}
 	}

@@ -171,8 +242,8 @@ func extractPublicKeyForOwner(i withPublicKey, forOwner *url.URL) (*rsa.PublicKe
 		return nil, nil, errors.New("public key property was nil")
 	}

-	for publicKeyIter := publicKeyProp.Begin(); publicKeyIter != publicKeyProp.End(); publicKeyIter = publicKeyIter.Next() {
-		pkey := publicKeyIter.Get()
+	for iter := publicKeyProp.Begin(); iter != publicKeyProp.End(); iter = iter.Next() {
+		pkey := iter.Get()
 		if pkey == nil {
 			continue
 		}
@@ -214,3 +285,263 @@ func extractPublicKeyForOwner(i withPublicKey, forOwner *url.URL) (*rsa.PublicKe
 	}
 	return nil, nil, errors.New("couldn't find public key")
 }
+
+func extractContent(i withContent) (string, error) {
+	contentProperty := i.GetActivityStreamsContent()
+	if contentProperty == nil {
+		return "", errors.New("content property was nil")
+	}
+	for iter := contentProperty.Begin(); iter != contentProperty.End(); iter = iter.Next() {
+		if iter.IsXMLSchemaString() && iter.GetXMLSchemaString() != "" {
+			return iter.GetXMLSchemaString(), nil
+		}
+	}
+	return "", errors.New("no content found")
+}
+
+func extractAttachments(i withAttachment) ([]*gtsmodel.MediaAttachment, error) {
+	attachments := []*gtsmodel.MediaAttachment{}
+
+	attachmentProp := i.GetActivityStreamsAttachment()
+	for iter := attachmentProp.Begin(); iter != attachmentProp.End(); iter = iter.Next() {
+		attachmentable, ok := iter.(Attachmentable)
+		if !ok {
+			continue
+		}
+		attachment, err := extractAttachment(attachmentable)
+		if err != nil {
+			continue
+		}
+		attachments = append(attachments, attachment)
+	}
+	return attachments, nil
+}
+
+func extractAttachment(i Attachmentable) (*gtsmodel.MediaAttachment, error) {
+	attachment := &gtsmodel.MediaAttachment{
+		File: gtsmodel.File{},
+	}
+
+	attachmentURL, err := extractURL(i)
+	if err != nil {
+		return nil, err
+	}
+	attachment.RemoteURL = attachmentURL.String()
+
+	mediaType := i.GetActivityStreamsMediaType()
+	if mediaType == nil {
+		return nil, errors.New("no media type")
+	}
+	if mediaType.Get() == "" {
+		return nil, errors.New("no media type")
+	}
+	attachment.File.ContentType = mediaType.Get()
+	attachment.Type = gtsmodel.FileTypeImage
+
+	name, err := extractName(i)
+	if err == nil {
+		attachment.Description = name
+	}
+
+	blurhash, err := extractBlurhash(i)
+	if err == nil {
+		attachment.Blurhash = blurhash
+	}
+
+	return attachment, nil
+}
+
+func extractBlurhash(i withBlurhash) (string, error) {
+	if i.GetTootBlurhashProperty() == nil {
+		return "", errors.New("blurhash property was nil")
+	}
+	if i.GetTootBlurhashProperty().Get() == "" {
+		return "", errors.New("empty blurhash string")
+	}
+	return i.GetTootBlurhashProperty().Get(), nil
+}
+
+func extractHashtags(i withTag) ([]*gtsmodel.Tag, error) {
+	tags := []*gtsmodel.Tag{}
+
+	tagsProp := i.GetActivityStreamsTag()
+	for iter := tagsProp.Begin(); iter != tagsProp.End(); iter = iter.Next() {
+		t := iter.GetType()
+		if t == nil {
+			continue
+		}
+
+		if t.GetTypeName() != "Hashtag" {
+			continue
+		}
+
+		hashtaggable, ok := t.(Hashtaggable)
+		if !ok {
+			continue
+		}
+
+		tag, err := extractHashtag(hashtaggable)
+		if err != nil {
+			continue
+		}
+
+		tags = append(tags, tag)
+	}
+	return tags, nil
+}
+
+func extractHashtag(i Hashtaggable) (*gtsmodel.Tag, error) {
+	tag := &gtsmodel.Tag{}
+
+	hrefProp := i.GetActivityStreamsHref()
+	if hrefProp == nil || !hrefProp.IsIRI() {
+		return nil, errors.New("no href prop")
+	}
+	tag.URL = hrefProp.GetIRI().String()
+
+	name, err := extractName(i)
+	if err != nil {
+		return nil, err
+	}
+	tag.Name = strings.TrimPrefix(name, "#")
+
+	return tag, nil
+}
+
+func extractEmojis(i withTag) ([]*gtsmodel.Emoji, error) {
+	emojis := []*gtsmodel.Emoji{}
+	tagsProp := i.GetActivityStreamsTag()
+	for iter := tagsProp.Begin(); iter != tagsProp.End(); iter = iter.Next() {
+		t := iter.GetType()
+		if t == nil {
+			continue
+		}
+
+		if t.GetTypeName() != "Emoji" {
+			continue
+		}
+
+		emojiable, ok := t.(Emojiable)
+		if !ok {
+			continue
+		}
+
+		emoji, err := extractEmoji(emojiable)
+		if err != nil {
+			continue
+		}
+
+		emojis = append(emojis, emoji)
+	}
+	return emojis, nil
+}
+
+func extractEmoji(i Emojiable) (*gtsmodel.Emoji, error) {
+	emoji := &gtsmodel.Emoji{}
+
+	idProp := i.GetJSONLDId()
+	if idProp == nil || !idProp.IsIRI() {
+		return nil, errors.New("no id for emoji")
+	}
+	uri := idProp.GetIRI()
+	emoji.URI = uri.String()
+	emoji.Domain = uri.Host
+
+	name, err := extractName(i)
+	if err != nil {
+		return nil, err
+	}
+	emoji.Shortcode = strings.Trim(name, ":")
+
+	if i.GetActivityStreamsIcon() == nil {
+		return nil, errors.New("no icon for emoji")
+	}
+	imageURL, err := extractIconURL(i)
+	if err != nil {
+		return nil, errors.New("no url for emoji image")
+	}
+	emoji.ImageRemoteURL = imageURL.String()
+
+	return emoji, nil
+}
+
+func extractMentions(i withTag) ([]*gtsmodel.Mention, error) {
+	mentions := []*gtsmodel.Mention{}
+	tagsProp := i.GetActivityStreamsTag()
+	for iter := tagsProp.Begin(); iter != tagsProp.End(); iter = iter.Next() {
+		t := iter.GetType()
+		if t == nil {
+			continue
+		}
+
+		if t.GetTypeName() != "Mention" {
+			continue
+		}
+
+		mentionable, ok := t.(Mentionable)
+		if !ok {
+			continue
+		}
+
+		mention, err := extractMention(mentionable)
+		if err != nil {
+			continue
+		}
+
+		mentions = append(mentions, mention)
+	}
+	return mentions, nil
+}
+
+func extractMention(i Mentionable) (*gtsmodel.Mention, error) {
+	mention := &gtsmodel.Mention{}
+
+	mentionString, err := extractName(i)
+	if err != nil {
+		return nil, err
+	}
+
+	// just make sure the mention string is valid so we can handle it properly later on...
+	username, domain, err := util.ExtractMentionParts(mentionString)
+	if err != nil {
+		return nil, err
+	}
+	if username == "" || domain == "" {
+		return nil, errors.New("username or domain was empty")
+	}
+	mention.NameString = mentionString
+
+	// the href prop should be the AP URI of a user we know, eg https://example.org/users/whatever_user
+	hrefProp := i.GetActivityStreamsHref()
+	if hrefProp == nil || !hrefProp.IsIRI() {
+		return nil, errors.New("no href prop")
+	}
+	mention.MentionedAccountURI = hrefProp.GetIRI().String()
+	return mention, nil
+}
+
+func extractActor(i withActor) (*url.URL, error) {
+	actorProp := i.GetActivityStreamsActor()
+	if actorProp == nil {
+		return nil, errors.New("actor property was nil")
+	}
+	for iter := actorProp.Begin(); iter != actorProp.End(); iter = iter.Next() {
+		if iter.IsIRI() && iter.GetIRI() != nil {
+			return iter.GetIRI(), nil
+		}
+	}
+	return nil, errors.New("no iri found for actor prop")
+}
+
+func extractObject(i withObject) (*url.URL, error) {
+	objectProp := i.GetActivityStreamsObject()
+	if objectProp == nil {
+		return nil, errors.New("object property was nil")
+	}
+	for iter := objectProp.Begin(); iter != objectProp.End(); iter = iter.Next() {
+		if iter.IsIRI() && iter.GetIRI() != nil {
+			return iter.GetIRI(), nil
+		}
+	}
+	return nil, errors.New("no iri found for object prop")
+}
--- a/internal/typeutils/asinterfaces.go
+++ b/internal/typeutils/asinterfaces.go
@@ -0,0 +1,237 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package typeutils
+
+import "github.com/go-fed/activity/streams/vocab"
+
+// Accountable represents the minimum activitypub interface for representing an 'account'.
+// This interface is fulfilled by: Person, Application, Organization, Service, and Group
+type Accountable interface {
+	withJSONLDId
+	withTypeName
+
+	withPreferredUsername
+	withIcon
+	withName
+	withImage
+	withSummary
+	withDiscoverable
+	withURL
+	withPublicKey
+	withInbox
+	withOutbox
+	withFollowing
+	withFollowers
+	withFeatured
+}
+
+// Statusable represents the minimum activitypub interface for representing a 'status'.
+// This interface is fulfilled by: Article, Document, Image, Video, Note, Page, Event, Place, Mention, Profile
+type Statusable interface {
+	withJSONLDId
+	withTypeName
+
+	withSummary
+	withInReplyTo
+	withPublished
+	withURL
+	withAttributedTo
+	withTo
+	withCC
+	withSensitive
+	withConversation
+	withContent
+	withAttachment
+	withTag
+	withReplies
+}
+
+// Attachmentable represents the minimum activitypub interface for representing a 'mediaAttachment'.
+// This interface is fulfilled by: Audio, Document, Image, Video
+type Attachmentable interface {
+	withTypeName
+	withMediaType
+	withURL
+	withName
+	withBlurhash
+	withFocalPoint
+}
+
+// Hashtaggable represents the minimum activitypub interface for representing a 'hashtag' tag.
+type Hashtaggable interface {
+	withTypeName
+	withHref
+	withName
+}
+
+// Emojiable represents the minimum interface for an 'emoji' tag.
+type Emojiable interface {
+	withJSONLDId
+	withTypeName
+	withName
+	withUpdated
+	withIcon
+}
+
+// Mentionable represents the minimum interface for a 'mention' tag.
+type Mentionable interface {
+	withName
+	withHref
+}
+
+// Followable represents the minimum interface for an activitystreams 'follow' activity.
+type Followable interface {
+	withJSONLDId
+	withTypeName
+
+	withActor
+	withObject
+}
+
+type withJSONLDId interface {
+	GetJSONLDId() vocab.JSONLDIdProperty
+}
+
+type withTypeName interface {
+	GetTypeName() string
+}
+
+type withPreferredUsername interface {
+	GetActivityStreamsPreferredUsername() vocab.ActivityStreamsPreferredUsernameProperty
+}
+
+type withIcon interface {
+	GetActivityStreamsIcon() vocab.ActivityStreamsIconProperty
+}
+
+type withName interface {
+	GetActivityStreamsName() vocab.ActivityStreamsNameProperty
+}
+
+type withImage interface {
+	GetActivityStreamsImage() vocab.ActivityStreamsImageProperty
+}
+
+type withSummary interface {
+	GetActivityStreamsSummary() vocab.ActivityStreamsSummaryProperty
+}
+
+type withDiscoverable interface {
+	GetTootDiscoverable() vocab.TootDiscoverableProperty
+}
+
+type withURL interface {
+	GetActivityStreamsUrl() vocab.ActivityStreamsUrlProperty
+}
+
+type withPublicKey interface {
+	GetW3IDSecurityV1PublicKey() vocab.W3IDSecurityV1PublicKeyProperty
+}
+
+type withInbox interface {
+	GetActivityStreamsInbox() vocab.ActivityStreamsInboxProperty
+}
+
+type withOutbox interface {
+	GetActivityStreamsOutbox() vocab.ActivityStreamsOutboxProperty
+}
+
+type withFollowing interface {
+	GetActivityStreamsFollowing() vocab.ActivityStreamsFollowingProperty
+}
+
+type withFollowers interface {
+	GetActivityStreamsFollowers() vocab.ActivityStreamsFollowersProperty
+}
+
+type withFeatured interface {
+	GetTootFeatured() vocab.TootFeaturedProperty
+}
+
+type withAttributedTo interface {
+	GetActivityStreamsAttributedTo() vocab.ActivityStreamsAttributedToProperty
+}
+
+type withAttachment interface {
+	GetActivityStreamsAttachment() vocab.ActivityStreamsAttachmentProperty
+}
+
+type withTo interface {
+	GetActivityStreamsTo() vocab.ActivityStreamsToProperty
+}
+
+type withInReplyTo interface {
+	GetActivityStreamsInReplyTo() vocab.ActivityStreamsInReplyToProperty
+}
+
+type withCC interface {
+	GetActivityStreamsCc() vocab.ActivityStreamsCcProperty
+}
+
+type withSensitive interface {
+	// TODO
+}
+
+type withConversation interface {
+	// TODO
+}
+
+type withContent interface {
+	GetActivityStreamsContent() vocab.ActivityStreamsContentProperty
+}
+
+type withPublished interface {
+	GetActivityStreamsPublished() vocab.ActivityStreamsPublishedProperty
+}
+
+type withTag interface {
+	GetActivityStreamsTag() vocab.ActivityStreamsTagProperty
+}
+
+type withReplies interface {
+	GetActivityStreamsReplies() vocab.ActivityStreamsRepliesProperty
+}
+
+type withMediaType interface {
+	GetActivityStreamsMediaType() vocab.ActivityStreamsMediaTypeProperty
+}
+
+type withBlurhash interface {
+	GetTootBlurhashProperty() vocab.TootBlurhashProperty
+}
+
+type withFocalPoint interface {
+	// TODO
+}
+
+type withHref interface {
+	GetActivityStreamsHref() vocab.ActivityStreamsHrefProperty
+}
+
+type withUpdated interface {
+	GetActivityStreamsUpdated() vocab.ActivityStreamsUpdatedProperty
+}
+
+type withActor interface {
+	GetActivityStreamsActor() vocab.ActivityStreamsActorProperty
+}
+
+type withObject interface {
+	GetActivityStreamsObject() vocab.ActivityStreamsObjectProperty
+}
--- a/internal/typeutils/astointernal.go
+++ b/internal/typeutils/astointernal.go
@@ -21,6 +21,8 @@ package typeutils
 import (
 	"errors"
 	"fmt"
+	"net/url"
+	"strings"

 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
@@ -157,3 +159,202 @@ func (c *converter) ASRepresentationToAccount(accountable Accountable) (*gtsmode

 	return acct, nil
 }
+
+func (c *converter) ASStatusToStatus(statusable Statusable) (*gtsmodel.Status, error) {
+	status := &gtsmodel.Status{}
+
+	// uri at which this status is reachable
+	uriProp := statusable.GetJSONLDId()
+	if uriProp == nil || !uriProp.IsIRI() {
+		return nil, errors.New("no id property found, or id was not an iri")
+	}
+	status.URI = uriProp.GetIRI().String()
+
+	// web url for viewing this status
+	if statusURL, err := extractURL(statusable); err == nil {
+		status.URL = statusURL.String()
+	}
+
+	// the html-formatted content of this status
+	if content, err := extractContent(statusable); err == nil {
+		status.Content = content
+	}
+
+	// attachments to dereference and fetch later on (we don't do that here)
+	if attachments, err := extractAttachments(statusable); err == nil {
+		status.GTSMediaAttachments = attachments
+	}
+
+	// hashtags to dereference later on
+	if hashtags, err := extractHashtags(statusable); err == nil {
+		status.GTSTags = hashtags
+	}
+
+	// emojis to dereference and fetch later on
+	if emojis, err := extractEmojis(statusable); err == nil {
+		status.GTSEmojis = emojis
+	}
+
+	// mentions to dereference later on
+	if mentions, err := extractMentions(statusable); err == nil {
+		status.GTSMentions = mentions
+	}
+
+	// cw string for this status
+	if cw, err := extractSummary(statusable); err == nil {
+		status.ContentWarning = cw
+	}
+
+	// when was this status created?
+	published, err := extractPublished(statusable)
+	if err == nil {
+		status.CreatedAt = published
+	}
+
+	// which account posted this status?
+	// if we don't know the account yet we can dereference it later
+	attributedTo, err := extractAttributedTo(statusable)
+	if err != nil {
+		return nil, errors.New("attributedTo was empty")
+	}
+	status.APStatusOwnerURI = attributedTo.String()
+
+	statusOwner := &gtsmodel.Account{}
+	if err := c.db.GetWhere("uri", attributedTo.String(), statusOwner); err != nil {
+		return nil, fmt.Errorf("couldn't get status owner from db: %s", err)
+	}
+	status.AccountID = statusOwner.ID
+	status.GTSAccount = statusOwner
+
+	// check if there's a post that this is a reply to
+	inReplyToURI, err := extractInReplyToURI(statusable)
+	if err == nil {
+		// something is set so we can at least set this field on the
+		// status and dereference using this later if we need to
+		status.APReplyToStatusURI = inReplyToURI.String()
+
+		// now we can check if we have the replied-to status in our db already
+		inReplyToStatus := &gtsmodel.Status{}
+		if err := c.db.GetWhere("uri", inReplyToURI.String(), inReplyToStatus); err == nil {
+			// we have the status in our database already
+			// so we can set these fields here and then...
+			status.InReplyToID = inReplyToStatus.ID
+			status.InReplyToAccountID = inReplyToStatus.AccountID
+			status.GTSReplyToStatus = inReplyToStatus
+
+			// ... check if we've seen the account already
+			inReplyToAccount := &gtsmodel.Account{}
+			if err := c.db.GetByID(inReplyToStatus.AccountID, inReplyToAccount); err == nil {
+				status.GTSReplyToAccount = inReplyToAccount
+			}
+		}
+	}
+
+	// visibility entry for this status
+	var visibility gtsmodel.Visibility
+
+	to, err := extractTos(statusable)
+	if err != nil {
+		return nil, fmt.Errorf("error extracting TO values: %s", err)
+	}
+
+	cc, err := extractCCs(statusable)
+	if err != nil {
+		return nil, fmt.Errorf("error extracting CC values: %s", err)
+	}
+
+	if len(to) == 0 && len(cc) == 0 {
+		return nil, errors.New("message wasn't TO or CC anyone")
+	}
+
+	// for visibility derivation, we start by assuming most restrictive, and work our way to least restrictive
+
+	// if it's a DM then it's addressed to SPECIFIC ACCOUNTS and not followers or public
+	if len(to) != 0 && len(cc) == 0 {
+		visibility = gtsmodel.VisibilityDirect
+	}
+
+	// if it's just got followers in TO and it's not also CC'ed to public, it's followers only
+	if isFollowers(to, statusOwner.FollowersURI) {
+		visibility = gtsmodel.VisibilityFollowersOnly
+	}
+
+	// if it's CC'ed to public, it's public or unlocked
+	// mentioned SPECIFIC ACCOUNTS also get added to CC'es if it's not a direct message
+	if isPublic(to) {
+		visibility = gtsmodel.VisibilityPublic
+	}
+
+	// we should have a visibility by now
+	if visibility == "" {
+		return nil, errors.New("couldn't derive visibility")
+	}
+	status.Visibility = visibility
+
+	// advanced visibility for this status
+	// TODO: a lot of work to be done here -- a new type needs to be created for this in go-fed/activity using ASTOOL
+
+	// sensitive
+	// TODO: this is a bool
+
+	// language
+	// we might be able to extract this from the contentMap field
+
+	// ActivityStreamsType
+	status.ActivityStreamsType = gtsmodel.ActivityStreamsObject(statusable.GetTypeName())
+
+	return status, nil
+}
+
+func (c *converter) ASFollowToFollowRequest(followable Followable) (*gtsmodel.FollowRequest, error) {
+
+	idProp := followable.GetJSONLDId()
+	if idProp == nil || !idProp.IsIRI() {
+		return nil, errors.New("no id property set on follow, or was not an iri")
+	}
+	uri := idProp.GetIRI().String()
+
+	origin, err := extractActor(followable)
+	if err != nil {
+		return nil, errors.New("error extracting actor property from follow")
+	}
+	originAccount := &gtsmodel.Account{}
+	if err := c.db.GetWhere("uri", origin.String(), originAccount); err != nil {
+		return nil, fmt.Errorf("error extracting account with uri %s from the database: %s", origin.String(), err)
+	}
+
+	target, err := extractObject(followable)
+	if err != nil {
+		return nil, errors.New("error extracting object property from follow")
+	}
+	targetAccount := &gtsmodel.Account{}
+	if err := c.db.GetWhere("uri", target.String(), targetAccount); err != nil {
+		return nil, fmt.Errorf("error extracting account with uri %s from the database: %s", origin.String(), err)
+	}
+
+	followRequest := &gtsmodel.FollowRequest{
+		URI:             uri,
+		AccountID:       originAccount.ID,
+		TargetAccountID: targetAccount.ID,
+	}
+
+	return followRequest, nil
+}
+
+func isPublic(tos []*url.URL) bool {
+	for _, entry := range tos {
+		if strings.EqualFold(entry.String(), "https://www.w3.org/ns/activitystreams#Public") {
+			return true
+		}
+	}
+	return false
+}
+
+func isFollowers(ccs []*url.URL, followersURI string) bool {
+	for _, entry := range ccs {
+		if strings.EqualFold(entry.String(), followersURI) {
+			return true
+		}
+	}
+	return false
+}
--- a/internal/typeutils/astointernal_test.go
+++ b/internal/typeutils/astointernal_test.go
@@ -25,6 +25,7 @@ import (
 	"testing"

 	"github.com/go-fed/activity/streams"
+	"github.com/go-fed/activity/streams/vocab"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/suite"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
@@ -36,6 +37,182 @@ type ASToInternalTestSuite struct {
 }

 const (
+	statusWithMentionsActivityJson = `{
+		"@context": [
+		  "https://www.w3.org/ns/activitystreams",
+		  {
+			"ostatus": "http://ostatus.org#",
+			"atomUri": "ostatus:atomUri",
+			"inReplyToAtomUri": "ostatus:inReplyToAtomUri",
+			"conversation": "ostatus:conversation",
+			"sensitive": "as:sensitive",
+			"toot": "http://joinmastodon.org/ns#",
+			"votersCount": "toot:votersCount"
+		  }
+		],
+		"id": "https://ondergrond.org/users/dumpsterqueer/statuses/106221634728637552/activity",
+		"type": "Create",
+		"actor": "https://ondergrond.org/users/dumpsterqueer",
+		"published": "2021-05-12T09:58:38Z",
+		"to": [
+		  "https://ondergrond.org/users/dumpsterqueer/followers"
+		],
+		"cc": [
+		  "https://www.w3.org/ns/activitystreams#Public",
+		  "https://social.pixie.town/users/f0x"
+		],
+		"object": {
+		  "id": "https://ondergrond.org/users/dumpsterqueer/statuses/106221634728637552",
+		  "type": "Note",
+		  "summary": null,
+		  "inReplyTo": "https://social.pixie.town/users/f0x/statuses/106221628567855262",
+		  "published": "2021-05-12T09:58:38Z",
+		  "url": "https://ondergrond.org/@dumpsterqueer/106221634728637552",
+		  "attributedTo": "https://ondergrond.org/users/dumpsterqueer",
+		  "to": [
+			"https://ondergrond.org/users/dumpsterqueer/followers"
+		  ],
+		  "cc": [
+			"https://www.w3.org/ns/activitystreams#Public",
+			"https://social.pixie.town/users/f0x"
+		  ],
+		  "sensitive": false,
+		  "atomUri": "https://ondergrond.org/users/dumpsterqueer/statuses/106221634728637552",
+		  "inReplyToAtomUri": "https://social.pixie.town/users/f0x/statuses/106221628567855262",
+		  "conversation": "tag:ondergrond.org,2021-05-12:objectId=1132361:objectType=Conversation",
+		  "content": "<p><span class=\"h-card\"><a href=\"https://social.pixie.town/@f0x\" class=\"u-url mention\">@<span>f0x</span></a></span> nice there it is:</p><p><a href=\"https://social.pixie.town/users/f0x/statuses/106221628567855262/activity\" rel=\"nofollow noopener noreferrer\" target=\"_blank\"><span class=\"invisible\">https://</span><span class=\"ellipsis\">social.pixie.town/users/f0x/st</span><span class=\"invisible\">atuses/106221628567855262/activity</span></a></p>",
+		  "contentMap": {
+			"en": "<p><span class=\"h-card\"><a href=\"https://social.pixie.town/@f0x\" class=\"u-url mention\">@<span>f0x</span></a></span> nice there it is:</p><p><a href=\"https://social.pixie.town/users/f0x/statuses/106221628567855262/activity\" rel=\"nofollow noopener noreferrer\" target=\"_blank\"><span class=\"invisible\">https://</span><span class=\"ellipsis\">social.pixie.town/users/f0x/st</span><span class=\"invisible\">atuses/106221628567855262/activity</span></a></p>"
+		  },
+		  "attachment": [],
+		  "tag": [
+			{
+			  "type": "Mention",
+			  "href": "https://social.pixie.town/users/f0x",
+			  "name": "@f0x@pixie.town"
+			}
+		  ],
+		  "replies": {
+			"id": "https://ondergrond.org/users/dumpsterqueer/statuses/106221634728637552/replies",
+			"type": "Collection",
+			"first": {
+			  "type": "CollectionPage",
+			  "next": "https://ondergrond.org/users/dumpsterqueer/statuses/106221634728637552/replies?only_other_accounts=true&page=true",
+			  "partOf": "https://ondergrond.org/users/dumpsterqueer/statuses/106221634728637552/replies",
+			  "items": []
+			}
+		  }
+		}
+	  }`
+	statusWithEmojisAndTagsAsActivityJson = `{
+		"@context": [
+		  "https://www.w3.org/ns/activitystreams",
+		  {
+			"ostatus": "http://ostatus.org#",
+			"atomUri": "ostatus:atomUri",
+			"inReplyToAtomUri": "ostatus:inReplyToAtomUri",
+			"conversation": "ostatus:conversation",
+			"sensitive": "as:sensitive",
+			"toot": "http://joinmastodon.org/ns#",
+			"votersCount": "toot:votersCount",
+			"Hashtag": "as:Hashtag",
+			"Emoji": "toot:Emoji",
+			"focalPoint": {
+			  "@container": "@list",
+			  "@id": "toot:focalPoint"
+			}
+		  }
+		],
+		"id": "https://ondergrond.org/users/dumpsterqueer/statuses/106221567884565704/activity",
+		"type": "Create",
+		"actor": "https://ondergrond.org/users/dumpsterqueer",
+		"published": "2021-05-12T09:41:38Z",
+		"to": [
+		  "https://ondergrond.org/users/dumpsterqueer/followers"
+		],
+		"cc": [
+		  "https://www.w3.org/ns/activitystreams#Public"
+		],
+		"object": {
+		  "id": "https://ondergrond.org/users/dumpsterqueer/statuses/106221567884565704",
+		  "type": "Note",
+		  "summary": null,
+		  "inReplyTo": null,
+		  "published": "2021-05-12T09:41:38Z",
+		  "url": "https://ondergrond.org/@dumpsterqueer/106221567884565704",
+		  "attributedTo": "https://ondergrond.org/users/dumpsterqueer",
+		  "to": [
+			"https://ondergrond.org/users/dumpsterqueer/followers"
+		  ],
+		  "cc": [
+			"https://www.w3.org/ns/activitystreams#Public"
+		  ],
+		  "sensitive": false,
+		  "atomUri": "https://ondergrond.org/users/dumpsterqueer/statuses/106221567884565704",
+		  "inReplyToAtomUri": null,
+		  "conversation": "tag:ondergrond.org,2021-05-12:objectId=1132361:objectType=Conversation",
+		  "content": "<p>just testing activitypub representations of <a href=\"https://ondergrond.org/tags/tags\" class=\"mention hashtag\" rel=\"tag\">#<span>tags</span></a> and <a href=\"https://ondergrond.org/tags/emoji\" class=\"mention hashtag\" rel=\"tag\">#<span>emoji</span></a>  :party_parrot: :amaze: :blobsunglasses: </p><p>don&apos;t mind me....</p>",
+		  "contentMap": {
+			"en": "<p>just testing activitypub representations of <a href=\"https://ondergrond.org/tags/tags\" class=\"mention hashtag\" rel=\"tag\">#<span>tags</span></a> and <a href=\"https://ondergrond.org/tags/emoji\" class=\"mention hashtag\" rel=\"tag\">#<span>emoji</span></a>  :party_parrot: :amaze: :blobsunglasses: </p><p>don&apos;t mind me....</p>"
+		  },
+		  "attachment": [],
+		  "tag": [
+			{
+			  "type": "Hashtag",
+			  "href": "https://ondergrond.org/tags/tags",
+			  "name": "#tags"
+			},
+			{
+			  "type": "Hashtag",
+			  "href": "https://ondergrond.org/tags/emoji",
+			  "name": "#emoji"
+			},
+			{
+			  "id": "https://ondergrond.org/emojis/2390",
+			  "type": "Emoji",
+			  "name": ":party_parrot:",
+			  "updated": "2020-11-06T13:42:11Z",
+			  "icon": {
+				"type": "Image",
+				"mediaType": "image/gif",
+				"url": "https://ondergrond.org/system/custom_emojis/images/000/002/390/original/ef133aac7ab23341.gif"
+			  }
+			},
+			{
+			  "id": "https://ondergrond.org/emojis/2395",
+			  "type": "Emoji",
+			  "name": ":amaze:",
+			  "updated": "2020-09-26T12:29:56Z",
+			  "icon": {
+				"type": "Image",
+				"mediaType": "image/png",
+				"url": "https://ondergrond.org/system/custom_emojis/images/000/002/395/original/2c7d9345e57367ed.png"
+			  }
+			},
+			{
+			  "id": "https://ondergrond.org/emojis/764",
+			  "type": "Emoji",
+			  "name": ":blobsunglasses:",
+			  "updated": "2020-09-26T12:13:23Z",
+			  "icon": {
+				"type": "Image",
+				"mediaType": "image/png",
+				"url": "https://ondergrond.org/system/custom_emojis/images/000/000/764/original/3f8eef9de773c90d.png"
+			  }
+			}
+		  ],
+		  "replies": {
+			"id": "https://ondergrond.org/users/dumpsterqueer/statuses/106221567884565704/replies",
+			"type": "Collection",
+			"first": {
+			  "type": "CollectionPage",
+			  "next": "https://ondergrond.org/users/dumpsterqueer/statuses/106221567884565704/replies?only_other_accounts=true&page=true",
+			  "partOf": "https://ondergrond.org/users/dumpsterqueer/statuses/106221567884565704/replies",
+			  "items": []
+			}
+		  }
+		}
+	  }`
 	gargronAsActivityJson = `{
 		"@context": [
 		  "https://www.w3.org/ns/activitystreams",
@@ -197,6 +374,62 @@ func (suite *ASToInternalTestSuite) TestParseGargron() {
 	// TODO: write assertions here, rn we're just eyeballing the output
 }

+func (suite *ASToInternalTestSuite) TestParseStatus() {
+	m := make(map[string]interface{})
+	err := json.Unmarshal([]byte(statusWithEmojisAndTagsAsActivityJson), &m)
+	assert.NoError(suite.T(), err)
+
+	t, err := streams.ToType(context.Background(), m)
+	assert.NoError(suite.T(), err)
+
+	create, ok := t.(vocab.ActivityStreamsCreate)
+	assert.True(suite.T(), ok)
+
+	obj := create.GetActivityStreamsObject()
+	assert.NotNil(suite.T(), obj)
+
+	first := obj.Begin()
+	assert.NotNil(suite.T(), first)
+
+	rep, ok := first.GetType().(typeutils.Statusable)
+	assert.True(suite.T(), ok)
+
+	status, err := suite.typeconverter.ASStatusToStatus(rep)
+	assert.NoError(suite.T(), err)
+
+	assert.Len(suite.T(), status.GTSEmojis, 3)
+	// assert.Len(suite.T(), status.GTSTags, 2) TODO: implement this first so that it can pick up tags
+}
+
+func (suite *ASToInternalTestSuite) TestParseStatusWithMention() {
+	m := make(map[string]interface{})
+	err := json.Unmarshal([]byte(statusWithMentionsActivityJson), &m)
+	assert.NoError(suite.T(), err)
+
+	t, err := streams.ToType(context.Background(), m)
+	assert.NoError(suite.T(), err)
+
+	create, ok := t.(vocab.ActivityStreamsCreate)
+	assert.True(suite.T(), ok)
+
+	obj := create.GetActivityStreamsObject()
+	assert.NotNil(suite.T(), obj)
+
+	first := obj.Begin()
+	assert.NotNil(suite.T(), first)
+
+	rep, ok := first.GetType().(typeutils.Statusable)
+	assert.True(suite.T(), ok)
+
+	status, err := suite.typeconverter.ASStatusToStatus(rep)
+	assert.NoError(suite.T(), err)
+
+	fmt.Printf("%+v", status)
+
+	assert.Len(suite.T(), status.GTSMentions, 1)
+	fmt.Println(status.GTSMentions[0])
+}
+
 func (suite *ASToInternalTestSuite) TearDownTest() {
 	testrig.StandardDBTeardown(suite.db)
 }
--- a/internal/typeutils/converter.go
+++ b/internal/typeutils/converter.go
@@ -90,6 +90,10 @@ type TypeConverter interface {

 	// ASPersonToAccount converts a remote account/person/application representation into a gts model account
 	ASRepresentationToAccount(accountable Accountable) (*gtsmodel.Account, error)
+	// ASStatus converts a remote activitystreams 'status' representation into a gts model status.
+	ASStatusToStatus(statusable Statusable) (*gtsmodel.Status, error)
+	// ASFollowToFollowRequest converts a remote activitystreams `follow` representation into gts model follow request.
+	ASFollowToFollowRequest(followable Followable) (*gtsmodel.FollowRequest, error)

 	/*
 		INTERNAL (gts) MODEL TO ACTIVITYSTREAMS MODEL
--- a/internal/typeutils/internaltoas.go
+++ b/internal/typeutils/internaltoas.go
@@ -200,15 +200,15 @@ func (c *converter) AccountToAS(a *gtsmodel.Account) (vocab.ActivityStreamsPerso
 	// icon
 	// Used as profile avatar.
 	if a.AvatarMediaAttachmentID != "" {
-		iconProperty := streams.NewActivityStreamsIconProperty()
-
-		iconImage := streams.NewActivityStreamsImage()
-
 		avatar := &gtsmodel.MediaAttachment{}
 		if err := c.db.GetByID(a.AvatarMediaAttachmentID, avatar); err != nil {
 			return nil, err
 		}

+		iconProperty := streams.NewActivityStreamsIconProperty()
+
+		iconImage := streams.NewActivityStreamsImage()
+
 		mediaType := streams.NewActivityStreamsMediaTypeProperty()
 		mediaType.Set(avatar.File.ContentType)
 		iconImage.SetActivityStreamsMediaType(mediaType)
@@ -228,15 +228,15 @@ func (c *converter) AccountToAS(a *gtsmodel.Account) (vocab.ActivityStreamsPerso
 	// image
 	// Used as profile header.
 	if a.HeaderMediaAttachmentID != "" {
-		headerProperty := streams.NewActivityStreamsImageProperty()
-
-		headerImage := streams.NewActivityStreamsImage()
-
 		header := &gtsmodel.MediaAttachment{}
 		if err := c.db.GetByID(a.HeaderMediaAttachmentID, header); err != nil {
 			return nil, err
 		}

+		headerProperty := streams.NewActivityStreamsImageProperty()
+
+		headerImage := streams.NewActivityStreamsImage()
+
 		mediaType := streams.NewActivityStreamsMediaTypeProperty()
 		mediaType.Set(header.File.ContentType)
 		headerImage.SetActivityStreamsMediaType(mediaType)