[feature] Update attachment format, receive + send focalPoint prop + use it on the frontend (#4052)

* [feature] Update attachment format, receive + send `focalPoint` prop + use it on the frontend

* whoops

* boop

* restore function signature of ExtractAttachments

docs/federation/posts.md

@@ -1,5 +1,56 @@
 # Posts and Post Properties
 
+## Attachments, Blurhash, and Focal Point
+
+GoToSocial sends media attachments in the `attachment` property of posts using the following types:
+
+- `Image` - any image type (webp, jpeg, gif, png, etc).
+- `Video` - any video type (mp4, mkv, webm, etc).
+- `Audio` - any audio type (mp3, flac, wma, etc).
+- `Document` - fallback for any other / unknown type.
+
+Attachments sent from GoToSocial include the MIME `mediaType`, the `url` of the full-sized version of the media file, and the `summary` property, which can be interpreted by remote instance's as a short description / alt text for the attachment.
+
+Types `Image` and `Video` will also include the `http://joinmastodon.org/ns#blurhash` property so that remotes can generate a colorful hash of the image. If an audio file included an embedded cover art image, then the `Audio` type will also include a blurhash. See the [Mastodon blurhash docs](https://docs.joinmastodon.org/spec/activitypub/#blurhash).
+
+`Image` types may also include the `http://joinmastodon.org/ns#focalPoint` property, which will be an array of two floats between -1.0 and 1.0 indicating the x-y coordinates of the image's focal point. See the [Mastondon focalPoint docs](https://docs.joinmastodon.org/spec/activitypub/#focalPoint).
+
+Here's an example of a `Note` with one attachment:
+
+```json
+{
+  "@context": [
+    "https://www.w3.org/ns/activitystreams",
+    {
+      "blurhash": "toot:blurhash",
+      "focalPoint": {
+        "@container": "@list",
+        "@id": "toot:focalPoint"
+      },
+      "toot": "http://joinmastodon.org/ns#"
+    }
+  ],
+  "type": "Note",
+  [...],
+  "attachment": [
+    {
+      "blurhash": "LIIE|gRj00WB-;j[t7j[4nWBj[Rj",
+      "focalPoint": [
+        -0.5,
+        0.5
+      ],
+      "mediaType": "image/jpeg",
+      "summary": "Black and white image of some 50's style text saying: Welcome On Board",
+      "type": "Image",
+      "url": "http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg"
+    }
+  ],
+  [...]
+}
+```
+
+When receiving posts with attachments from remote instances, it will try to parse any of the four types `Image`, `Video`, `Audio`, or `Document` into media attachments. It doesn't matter which type is used. It will check for `blurhash` and `focalPoint` properties and use these if they are set. It will use the `summary` value as a short description / alt text, falling back to `name` if `summary` is not set.
+
 ## Hashtags
 
 GoToSocial users can include hashtags in their posts, which indicate to other instances that that user wishes their post to be grouped together with other posts using the same hashtag, for discovery purposes.

internal/ap/extract.go

@@ -634,32 +634,38 @@ func ExtractContent(i WithContent) gtsmodel.Content {
 	return content
 }
 
-// ExtractAttachments attempts to extract barebones MediaAttachment objects from given AS interface type.
+// ExtractAttachments attempts to extract barebones
+// MediaAttachment objects from given AS interface type.
 func ExtractAttachments(i WithAttachment) ([]*gtsmodel.MediaAttachment, error) {
 	attachmentProp := i.GetActivityStreamsAttachment()
 	if attachmentProp == nil {
 		return nil, nil
 	}
 
-	var errs gtserror.MultiError
+	var (
+		attachments = make([]*gtsmodel.MediaAttachment, 0, attachmentProp.Len())
+		errs        gtserror.MultiError
+	)
 
-	attachments := make([]*gtsmodel.MediaAttachment, 0, attachmentProp.Len())
 	for iter := attachmentProp.Begin(); iter != attachmentProp.End(); iter = iter.Next() {
 		t := iter.GetType()
 		if t == nil {
 			errs.Appendf("nil attachment type")
 			continue
 		}
-		attachmentable, ok := t.(Attachmentable)
+
+		attachmentable, ok := ToAttachmentable(t)
 		if !ok {
-			errs.Appendf("incorrect attachment type: %T", t)
+			errs.Appendf("could not cast %T to Attachmentable", t)
 			continue
 		}
+
 		attachment, err := ExtractAttachment(attachmentable)
 		if err != nil {
 			errs.Appendf("error extracting attachment: %w", err)
 			continue
 		}
+
 		attachments = append(attachments, attachment)
 	}
 
@@ -681,6 +687,9 @@ func ExtractAttachment(i Attachmentable) (*gtsmodel.MediaAttachment, error) {
 		RemoteURL:   remoteURL.String(),
 		Description: ExtractDescription(i),
 		Blurhash:    ExtractBlurhash(i),
+		FileMeta: gtsmodel.FileMeta{
+			Focus: ExtractFocus(i),
+		},
 		Processing: gtsmodel.ProcessingStatusReceived,
 	}, nil
 }
@@ -708,6 +717,50 @@ func ExtractBlurhash(i WithBlurhash) string {
 	return blurhashProp.Get()
 }
 
+// ExtractFocus parses a gtsmodel.Focus from the given Attachmentable's
+// `focalPoint` property, if Attachmentable can have `focalPoint`, and
+// `focalPoint` is set to a valid pair of floats. Otherwise, returns a
+// zero gtsmodel.Focus (ie., focus in the centre of the image).
+func ExtractFocus(attachmentable Attachmentable) gtsmodel.Focus {
+	focus := gtsmodel.Focus{}
+
+	withFocalPoint, ok := attachmentable.(WithFocalPoint)
+	if !ok {
+		return focus
+	}
+
+	focalPointProp := withFocalPoint.GetTootFocalPoint()
+	if focalPointProp == nil || focalPointProp.Len() != 2 {
+		return focus
+	}
+
+	xProp := focalPointProp.At(0)
+	if !xProp.IsXMLSchemaFloat() {
+		return focus
+	}
+
+	yProp := focalPointProp.At(1)
+	if !yProp.IsXMLSchemaFloat() {
+		return focus
+	}
+
+	x := xProp.Get()
+	if x < -1 || x > 1 {
+		return focus
+	}
+
+	y := yProp.Get()
+	if y < -1 || y > 1 {
+		return focus
+	}
+
+	// Looks good.
+	focus.X = float32(x)
+	focus.Y = float32(y)
+
+	return focus
+}
+
 // ExtractHashtags extracts a slice of minimal gtsmodel.Tags
 // from a WithTag. If an entry in the WithTag is not a hashtag,
 // or has a name that cannot be normalized, it will be ignored.

internal/ap/extractfocus_test.go

@@ -0,0 +1,125 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package ap_test
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"testing"
+
+	"code.superseriousbusiness.org/activity/streams"
+	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/ap"
+)
+
+type ExtractFocusTestSuite struct {
+	APTestSuite
+}
+
+func (suite *ExtractFocusTestSuite) TestExtractFocus() {
+	ctx := context.Background()
+
+	type test struct {
+		data    string
+		expectX float32
+		expectY float32
+	}
+
+	for _, test := range []test{
+		{
+			// Fine.
+			data:    "-0.5, 0.5",
+			expectX: -0.5,
+			expectY: 0.5,
+		},
+		{
+			// Also fine.
+			data:    "1, 1",
+			expectX: 1,
+			expectY: 1,
+		},
+		{
+			// Out of range.
+			data:    "1.5, 1",
+			expectX: 0,
+			expectY: 0,
+		},
+		{
+			// Too many points.
+			data:    "1, 1, 0",
+			expectX: 0,
+			expectY: 0,
+		},
+		{
+			// Not enough points.
+			data:    "1",
+			expectX: 0,
+			expectY: 0,
+		},
+	} {
+		// Wrap provided test.data
+		// in a minimal Attachmentable.
+		const fmts = `{
+  "@context": [
+    "https://www.w3.org/ns/activitystreams",
+    {
+      "focalPoint": {
+        "@container": "@list",
+        "@id": "toot:focalPoint"
+      },
+      "toot": "http://joinmastodon.org/ns#"
+    }
+  ],
+  "focalPoint": [ %s ],
+  "type": "Image"
+}`
+
+		// Unmarshal test data.
+		data := fmt.Sprintf(fmts, test.data)
+		m := make(map[string]any)
+		if err := json.Unmarshal([]byte(data), &m); err != nil {
+			suite.FailNow(err.Error())
+		}
+
+		// Convert to type.
+		t, err := streams.ToType(ctx, m)
+		if err != nil {
+			suite.FailNow(err.Error())
+		}
+
+		// Convert to attachmentable.
+		attachmentable, ok := t.(ap.Attachmentable)
+		if !ok {
+			suite.FailNow("", "%T was not Attachmentable", t)
+		}
+
+		// Check extracted focus.
+		focus := ap.ExtractFocus(attachmentable)
+		if focus.X != test.expectX || focus.Y != test.expectY {
+			suite.Fail("",
+				"expected x=%.2f y=%.2f got x=%.2f y=%.2f",
+				test.expectX, test.expectY, focus.X, focus.Y,
+			)
+		}
+	}
+}
+
+func TestExtractFocusTestSuite(t *testing.T) {
+	suite.Run(t, new(ExtractFocusTestSuite))
+}

internal/ap/interfaces.go

@@ -165,6 +165,29 @@ func ToApprovable(t vocab.Type) (Approvable, bool) {
 	return approvable, true
 }
 
+// IsAttachmentable returns whether AS vocab type name
+// is something that can be cast to Attachmentable.
+func IsAttachmentable(typeName string) bool {
+	switch typeName {
+	case ObjectAudio,
+		ObjectDocument,
+		ObjectImage,
+		ObjectVideo:
+		return true
+	default:
+		return false
+	}
+}
+
+// ToAttachmentable safely tries to cast vocab.Type as Attachmentable.
+func ToAttachmentable(t vocab.Type) (Attachmentable, bool) {
+	attachmentable, ok := t.(Attachmentable)
+	if !ok || !IsAttachmentable(t.GetTypeName()) {
+		return nil, false
+	}
+	return attachmentable, true
+}
+
 // Activityable represents the minimum activitypub interface for representing an 'activity'.
 // (see: IsActivityable() for types implementing this, though you MUST make sure to check
 // the typeName as this bare interface may be implementable by non-Activityable types).
@@ -628,9 +651,11 @@ type WithBlurhash interface {
 	SetTootBlurhash(vocab.TootBlurhashProperty)
 }
 
-// type withFocalPoint interface {
+// WithFocalPoint represents an object with TootFocalPointProperty.
-// 	// TODO
+type WithFocalPoint interface {
-// }
+	GetTootFocalPoint() vocab.TootFocalPointProperty
+	SetTootFocalPoint(vocab.TootFocalPointProperty)
+}
 
 // WithHref represents an activity with ActivityStreamsHrefProperty
 type WithHref interface {

internal/ap/properties.go

@@ -560,6 +560,70 @@ func SetApprovedBy(with WithApprovedBy, approvedBy *url.URL) {
 	abProp.Set(approvedBy)
 }
 
+// GetMediaType returns the string contained in
+// the MediaType property of 'with', if set.
+func GetMediaType(with WithMediaType) string {
+	mtProp := with.GetActivityStreamsMediaType()
+	if mtProp == nil || !mtProp.IsRFCRfc2045() {
+		return ""
+	}
+	return mtProp.Get()
+}
+
+// SetMediaType sets the given string
+// on the MediaType property of 'with'.
+func SetMediaType(with WithMediaType, mediaType string) {
+	mtProp := with.GetActivityStreamsMediaType()
+	if mtProp == nil {
+		mtProp = streams.NewActivityStreamsMediaTypeProperty()
+		with.SetActivityStreamsMediaType(mtProp)
+	}
+	mtProp.Set(mediaType)
+}
+
+// AppendName appends the given name
+// vals to the Name property of 'with'.
+func AppendName(with WithName, name ...string) {
+	if len(name) == 0 {
+		return
+	}
+	nameProp := with.GetActivityStreamsName()
+	if nameProp == nil {
+		nameProp = streams.NewActivityStreamsNameProperty()
+		with.SetActivityStreamsName(nameProp)
+	}
+	for _, name := range name {
+		nameProp.AppendXMLSchemaString(name)
+	}
+}
+
+// AppendSummary appends the given summary
+// vals to the Summary property of 'with'.
+func AppendSummary(with WithSummary, summary ...string) {
+	if len(summary) == 0 {
+		return
+	}
+	summaryProp := with.GetActivityStreamsSummary()
+	if summaryProp == nil {
+		summaryProp = streams.NewActivityStreamsSummaryProperty()
+		with.SetActivityStreamsSummary(summaryProp)
+	}
+	for _, summary := range summary {
+		summaryProp.AppendXMLSchemaString(summary)
+	}
+}
+
+// SetBlurhash sets the given string
+// on the Blurhash property of 'with'.
+func SetBlurhash(with WithBlurhash, mediaType string) {
+	bProp := with.GetTootBlurhash()
+	if bProp == nil {
+		bProp = streams.NewTootBlurhashProperty()
+		with.SetTootBlurhash(bProp)
+	}
+	bProp.Set(mediaType)
+}
+
 // extractIRIs extracts just the AP IRIs from an iterable
 // property that may contain types (with IRIs) or just IRIs.
 //

internal/api/client/statuses/statusboost_test.go

@@ -193,8 +193,8 @@ func (suite *StatusBoostTestSuite) TestPostBoost() {
         "id": "01F8MH6NEM8D7527KZAECTCR76",
         "meta": {
           "focus": {
-            "x": 0,
+            "x": -0.5,
-            "y": 0
+            "y": 0.5
           },
           "original": {
             "aspect": 1.9047619,

internal/federation/dereferencing/status.go

@@ -950,6 +950,8 @@ func (d *Dereferencer) fetchStatusAttachments(
 				RemoteURL:   &placeholder.RemoteURL,
 				Description: &placeholder.Description,
 				Blurhash:    &placeholder.Blurhash,
+				FocusX:      &placeholder.FileMeta.Focus.X,
+				FocusY:      &placeholder.FileMeta.Focus.Y,
 			},
 		)
 		if err != nil {

internal/filter/spam/statusable.go

@@ -142,7 +142,14 @@ func (f *Filter) StatusableOK(
 	}
 
 	// HEURISTIC 6: Are there any media attachments?
-	attachments, _ := ap.ExtractAttachments(statusable)
+	attachments, err := ap.ExtractAttachments(statusable)
+	if err != nil {
+		log.Warnf(ctx,
+			"error(s) extracting attachments for %s: %v",
+			ap.GetJSONLDId(statusable), err,
+		)
+	}
+
 	hasAttachments := len(attachments) != 0
 	if hasAttachments {
 		err := errors.New("status has attachment(s)")

internal/router/template.go

@@ -136,6 +136,7 @@ func LoadTemplates(engine *gin.Engine) error {
 var funcMap = template.FuncMap{
 	"add":              add,
 	"acctInstance":     acctInstance,
+	"objectPosition":   objectPosition,
 	"demojify":         demojify,
 	"deref":            deref,
 	"emojify":          emojify,
@@ -365,3 +366,12 @@ func deref(i any) any {
 
 	return vOf.Elem()
 }
+
+// objectPosition formats the given focus coordinates to a
+// string suitable for use as a css object-position value.
+func objectPosition(focusX float32, focusY float32) string {
+	const fmts = "%.2f"
+	xPos := ((focusX / 2) + .5) * 100
+	yPos := ((focusY / -2) + .5) * 100
+	return fmt.Sprintf(fmts, xPos) + "%" + " " + fmt.Sprintf(fmts, yPos) + "%"
+}

internal/typeutils/internaltoas.go

@@ -678,22 +678,9 @@ func (c *Converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (ap.Stat
 	status.SetActivityStreamsContent(contentProp)
 
 	// attachments
-	attachmentProp := streams.NewActivityStreamsAttachmentProperty()
+	if err := c.attachAttachments(ctx, s, status); err != nil {
-	attachments := s.Attachments
+		return nil, gtserror.Newf("error attaching attachments: %w", err)
-	if len(s.AttachmentIDs) != len(attachments) {
-		attachments, err = c.state.DB.GetAttachmentsByIDs(ctx, s.AttachmentIDs)
-		if err != nil {
-			return nil, gtserror.Newf("error getting attachments from database: %w", err)
 	}
-	}
-	for _, a := range attachments {
-		doc, err := c.AttachmentToAS(ctx, a)
-		if err != nil {
-			return nil, gtserror.Newf("error converting attachment: %w", err)
-		}
-		attachmentProp.AppendActivityStreamsDocument(doc)
-	}
-	status.SetActivityStreamsAttachment(attachmentProp)
 
 	// replies
 	repliesCollection, err := c.StatusToASRepliesCollection(ctx, s, false)
@@ -1130,39 +1117,94 @@ func (c *Converter) EmojiToAS(ctx context.Context, e *gtsmodel.Emoji) (vocab.Too
 	return emoji, nil
 }
 
-// AttachmentToAS converts a gts model media attachment into an activity streams Attachment, suitable for federation
+// attachAttachments converts the attachments on the given status
-func (c *Converter) AttachmentToAS(ctx context.Context, a *gtsmodel.MediaAttachment) (vocab.ActivityStreamsDocument, error) {
+// into Attachmentables, and appends them to the given Statusable.
-	// type -- Document
+func (c *Converter) attachAttachments(
-	doc := streams.NewActivityStreamsDocument()
+	ctx context.Context,
+	s *gtsmodel.Status,
+	statusable ap.Statusable,
+) error {
+	// Ensure status attachments populated.
+	if len(s.AttachmentIDs) != len(s.Attachments) {
+		var err error
+		s.Attachments, err = c.state.DB.GetAttachmentsByIDs(ctx, s.AttachmentIDs)
+		if err != nil && !errors.Is(err, db.ErrNoEntries) {
+			return gtserror.Newf("db error getting attachments: %w", err)
+		}
+	}
 
-	// mediaType aka mime content type
+	// Prepare attachment property.
-	mediaTypeProp := streams.NewActivityStreamsMediaTypeProperty()
+	attachmentProp := streams.NewActivityStreamsAttachmentProperty()
-	mediaTypeProp.Set(a.File.ContentType)
+	defer statusable.SetActivityStreamsAttachment(attachmentProp)
-	doc.SetActivityStreamsMediaType(mediaTypeProp)
 
-	// url -- for the original image not the thumbnail
+	for _, a := range s.Attachments {
-	urlProp := streams.NewActivityStreamsUrlProperty()
+
+		// Use appropriate vocab.Type and
+		// append function for this attachment.
+		var (
+			attachmentable ap.Attachmentable
+			append         func()
+		)
+		switch a.Type {
+
+		// png, gif, webp, jpeg, etc.
+		case gtsmodel.FileTypeImage:
+			t := streams.NewActivityStreamsImage()
+			attachmentable = t
+			append = func() { attachmentProp.AppendActivityStreamsImage(t) }
+
+		// mp4, m4a, wmv, webm, etc.
+		case gtsmodel.FileTypeVideo, gtsmodel.FileTypeGifv:
+			t := streams.NewActivityStreamsVideo()
+			attachmentable = t
+			append = func() { attachmentProp.AppendActivityStreamsVideo(t) }
+
+		// mp3, flac, ogg, wma, etc.
+		case gtsmodel.FileTypeAudio:
+			t := streams.NewActivityStreamsAudio()
+			attachmentable = t
+			append = func() { attachmentProp.AppendActivityStreamsAudio(t) }
+
+		// Not sure, fall back to Document.
+		default:
+			t := streams.NewActivityStreamsDocument()
+			attachmentable = t
+			append = func() { attachmentProp.AppendActivityStreamsDocument(t) }
+		}
+
+		// `mediaType` ie., mime content type.
+		ap.SetMediaType(attachmentable, a.File.ContentType)
+
+		// URL of the media file.
 		imageURL, err := url.Parse(a.URL)
 		if err != nil {
-		return nil, fmt.Errorf("AttachmentToAS: error parsing uri %s: %s", a.URL, err)
+			return gtserror.Newf("error parsing attachment url: %w", err)
 		}
-	urlProp.AppendIRI(imageURL)
+		ap.AppendURL(attachmentable, imageURL)
-	doc.SetActivityStreamsUrl(urlProp)
 
-	// name -- aka image description
+		// `summary` ie., media description / alt text
-	nameProp := streams.NewActivityStreamsNameProperty()
+		ap.AppendSummary(attachmentable, a.Description)
-	nameProp.AppendXMLSchemaString(a.Description)
-	doc.SetActivityStreamsName(nameProp)
 
-	// blurhash
+		// `blurhash`
-	blurProp := streams.NewTootBlurhashProperty()
+		ap.SetBlurhash(attachmentable, a.Blurhash)
-	blurProp.Set(a.Blurhash)
-	doc.SetTootBlurhash(blurProp)
 
-	// focalpoint
+		// Set `focalPoint` only if necessary.
-	// TODO
+		if a.FileMeta.Focus.X != 0 && a.FileMeta.Focus.Y != 0 {
+			if withFocalPoint, ok := attachmentable.(ap.WithFocalPoint); ok {
+				focalPointProp := streams.NewTootFocalPointProperty()
+				focalPointProp.AppendXMLSchemaFloat(float64(a.FileMeta.Focus.X))
+				focalPointProp.AppendXMLSchemaFloat(float64(a.FileMeta.Focus.Y))
+				withFocalPoint.SetTootFocalPoint(focalPointProp)
+			}
+		}
 
-	return doc, nil
+		// Done, append
+		// to Statusable.
+		append()
+	}
+
+	statusable.SetActivityStreamsAttachment(attachmentProp)
+	return nil
 }
 
 // FaveToAS converts a gts model status fave into an activityStreams LIKE, suitable for federation.

internal/typeutils/internaltoas_test.go

@@ -597,6 +597,10 @@ func (suite *InternalToASTestSuite) TestStatusWithTagsToASWithIDs() {
       "Emoji": "toot:Emoji",
       "Hashtag": "as:Hashtag",
       "blurhash": "toot:blurhash",
+      "focalPoint": {
+        "@container": "@list",
+        "@id": "toot:focalPoint"
+      },
       "sensitive": "as:sensitive",
       "toot": "http://joinmastodon.org/ns#"
     }
@@ -604,9 +608,13 @@ func (suite *InternalToASTestSuite) TestStatusWithTagsToASWithIDs() {
   "attachment": [
     {
       "blurhash": "LIIE|gRj00WB-;j[t7j[4nWBj[Rj",
+      "focalPoint": [
+        -0.5,
+        0.5
+      ],
       "mediaType": "image/jpeg",
-      "name": "Black and white image of some 50's style text saying: Welcome On Board",
+      "summary": "Black and white image of some 50's style text saying: Welcome On Board",
-      "type": "Document",
+      "type": "Image",
       "url": "http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg"
     }
   ],
@@ -697,6 +705,10 @@ func (suite *InternalToASTestSuite) TestStatusWithTagsToASFromDB() {
       "Emoji": "toot:Emoji",
       "Hashtag": "as:Hashtag",
       "blurhash": "toot:blurhash",
+      "focalPoint": {
+        "@container": "@list",
+        "@id": "toot:focalPoint"
+      },
       "sensitive": "as:sensitive",
       "toot": "http://joinmastodon.org/ns#"
     }
@@ -704,9 +716,13 @@ func (suite *InternalToASTestSuite) TestStatusWithTagsToASFromDB() {
   "attachment": [
     {
       "blurhash": "LIIE|gRj00WB-;j[t7j[4nWBj[Rj",
+      "focalPoint": [
+        -0.5,
+        0.5
+      ],
       "mediaType": "image/jpeg",
-      "name": "Black and white image of some 50's style text saying: Welcome On Board",
+      "summary": "Black and white image of some 50's style text saying: Welcome On Board",
-      "type": "Document",
+      "type": "Image",
       "url": "http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg"
     }
   ],

internal/typeutils/internaltofrontend_test.go

@@ -553,8 +553,8 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontend() {
           "aspect": 1.9104477
         },
         "focus": {
-          "x": 0,
+          "x": -0.5,
-          "y": 0
+          "y": 0.5
         }
       },
       "description": "Black and white image of some 50's style text saying: Welcome On Board",
@@ -701,8 +701,8 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendHTMLContentWarning
           "aspect": 1.9104477
         },
         "focus": {
-          "x": 0,
+          "x": -0.5,
-          "y": 0
+          "y": 0.5
         }
       },
       "description": "Black and white image of some 50's style text saying: Welcome On Board",
@@ -851,8 +851,8 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendApplicationDeleted
           "aspect": 1.9104477
         },
         "focus": {
-          "x": 0,
+          "x": -0.5,
-          "y": 0
+          "y": 0.5
         }
       },
       "description": "Black and white image of some 50's style text saying: Welcome On Board",
@@ -1032,8 +1032,8 @@ func (suite *InternalToFrontendTestSuite) TestWarnFilteredStatusToFrontend() {
           "aspect": 1.9104477
         },
         "focus": {
-          "x": 0,
+          "x": -0.5,
-          "y": 0
+          "y": 0.5
         }
       },
       "description": "Black and white image of some 50's style text saying: Welcome On Board",
@@ -1218,8 +1218,8 @@ func (suite *InternalToFrontendTestSuite) TestWarnFilteredBoostToFrontend() {
             "aspect": 1.9104477
           },
           "focus": {
-            "x": 0,
+            "x": -0.5,
-            "y": 0
+            "y": 0.5
           }
         },
         "description": "Black and white image of some 50's style text saying: Welcome On Board",
@@ -1955,8 +1955,8 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendUnknownLanguage()
           "aspect": 1.9104477
         },
         "focus": {
-          "x": 0,
+          "x": -0.5,
-          "y": 0
+          "y": 0.5
         }
       },
       "description": "Black and white image of some 50's style text saying: Welcome On Board",

testrig/testmodels.go

@@ -709,6 +709,12 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
 					Size:   137216,
 					Aspect: 1.9104477,
 				},
+				// Focus on top-left
+				// quadrant of image.
+				Focus: gtsmodel.Focus{
+					X: -0.5,
+					Y: 0.5,
+				},
 			},
 			AccountID:         "01F8MH17FWEB39HZJ76B6VXSKF",
 			Description:       "Black and white image of some 50's style text saying: Welcome On Board",

web/source/frontend/index.js

@@ -29,6 +29,7 @@
 const Photoswipe = require("photoswipe/dist/umd/photoswipe.umd.min.js");
 const PhotoswipeLightbox = require("photoswipe/dist/umd/photoswipe-lightbox.umd.min.js");
 const PhotoswipeCaptionPlugin = require("photoswipe-dynamic-caption-plugin").default;
+const ObjectPosition = require("./photoswipe-object-position.js").default;
 const Plyr = require("plyr");
 const Prism = require("./prism.js");
 
@@ -61,6 +62,10 @@ new PhotoswipeCaptionPlugin(lightbox, {
 	}
 });
 
+// Enable object-position plugin for lightbox so that css
+// object-position property can be used on preview images.
+new ObjectPosition(lightbox);
+
 lightbox.addFilter('itemData', (item) => {
 	const el = item.element;
 	if (

web/source/frontend/photoswipe-object-position.js

@@ -0,0 +1,119 @@
+/*
+	GoToSocial
+	Copyright (C) GoToSocial Authors admin@gotosocial.org
+	SPDX-License-Identifier: AGPL-3.0-or-later
+
+	This program is free software: you can redistribute it and/or modify
+	it under the terms of the GNU Affero General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU Affero General Public License for more details.
+
+	You should have received a copy of the GNU Affero General Public License
+	along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+/*
+	Code in this file adapted from:
+	https://github.com/vovayatsyuk/photoswipe-object-position (MIT License).
+*/
+
+function getCroppedBoundsOffset(position, imageSize, thumbSize, zoomLevel) {
+	const float = parseFloat(position); 
+	return position.indexOf('%') > 0
+		? (thumbSize - imageSize * zoomLevel) * float / 100
+		: float;
+}
+ 
+function getCroppedZoomPan(position, min, max) {
+	const float = parseFloat(position);
+	return position.indexOf('%') > 0 ? min + (max - min) * float / 100 : float;
+}
+
+function getThumbnail(el) {
+	return el.querySelector('img');
+}
+
+function getObjectPosition(el) {
+	return getComputedStyle(el).getPropertyValue('object-position').split(' ');
+}
+
+export default class ObjectPosition {
+	constructor(lightbox) {
+		/**
+		* Make pan adjustments if large image doesn't fit the viewport.
+		*
+		* Examples:
+		* 1. When thumb object-position is 50% 0 (top part is initially visible)
+		*    make sure you'll see the top part of the large image as well.
+		* 2. When thumb object-position is 50% 100% (bottom part is initially visible)
+		*    make sure you'll see the bottom part of the large image as well.
+		*/
+		lightbox.on('initialZoomPan', (event) => {
+			const slide = event.slide;
+			if (!slide.data.element) {
+				// No thumbnail
+				// image set.
+				return;
+			}
+			
+			const thumbnailImg = getThumbnail(slide.data.element);
+			if (!thumbnailImg) {
+				// No thumbnail
+				// image set.
+				return;
+			}
+
+			const [positionX, positionY] = getObjectPosition(thumbnailImg);
+
+			if (positionX !== '50%' && slide.pan.x < 0) {
+				slide.pan.x = getCroppedZoomPan(positionX, slide.bounds.min.x, slide.bounds.max.x);
+			}
+ 
+			if (positionY !== '50%' && slide.pan.y < 0) {
+				slide.pan.y = getCroppedZoomPan(positionY, slide.bounds.min.y, slide.bounds.max.y);
+			}
+		});
+
+		/**
+		* Fix opening animation when thumb object-position is not 50% 50%.
+		* https://github.com/dimsemenov/PhotoSwipe/pull/1868
+		*/
+		lightbox.addFilter('thumbBounds', (thumbBounds, itemData) => {
+			if (!itemData.element) {
+				// No thumbnail
+				// image set.
+				return;
+			}
+			
+			const thumbnailImg = getThumbnail(itemData.element);
+			if (!thumbnailImg) {
+				// No thumbnail
+				// image set.
+				return;
+			}
+
+			const thumbAreaRect = thumbnailImg.getBoundingClientRect();
+			const fillZoomLevel = thumbBounds.w / itemData.width;
+			const [positionX, positionY] = getObjectPosition(thumbnailImg);
+
+			if (positionX !== '50%') {
+				const offsetX = getCroppedBoundsOffset(positionX, itemData.width, thumbAreaRect.width, fillZoomLevel);
+				thumbBounds.x = thumbAreaRect.left + offsetX;
+				thumbBounds.innerRect.x = offsetX;
+			}
+
+			if (positionY !== '50%') {
+				const offsetY = getCroppedBoundsOffset(positionY, itemData.height, thumbAreaRect.height, fillZoomLevel);
+				thumbBounds.y = thumbAreaRect.top + offsetY;
+				thumbBounds.innerRect.y = offsetY;
+			}
+
+			return thumbBounds;
+		});
+	}
+}

web/source/frontend_prerender/index.js

@@ -29,6 +29,11 @@ import { decode } from "blurhash";
 
 const reduceMotion = window.matchMedia('(prefers-reduced-motion: reduce)');
 
+// Adjust object-position of any image that has a focal point set.
+document.querySelectorAll("img[data-object-position]").forEach(img => {
+	img.style["object-position"] = img.dataset.objectPosition;
+});
+
 // Generate a blurhash canvas for each image for
 // each blurhash container and put it in the summary.
 Array.from(document.getElementsByClassName('blurhash-container')).forEach(blurhashContainer => {
@@ -36,6 +41,7 @@ Array.from(document.getElementsByClassName('blurhash-container')).forEach(blurha
 	const thumbHeight = blurhashContainer.dataset.blurhashHeight;
 	const thumbWidth = blurhashContainer.dataset.blurhashWidth;
 	const thumbAspect = blurhashContainer.dataset.blurhashAspect;	
+	const objectPosition = blurhashContainer.dataset.blurhashObjectPosition;
 	
 	/*
 		It's very expensive to draw big canvases
@@ -73,6 +79,12 @@ Array.from(document.getElementsByClassName('blurhash-container')).forEach(blurha
 	imageData.data.set(pixels);
 	ctx.putImageData(imageData, 0, 0);
 
+	// Set object-position css property on
+	// the canvas if it's set on the container.
+	if (objectPosition) {
+		canvas.style["object-position"] = objectPosition;
+	}
+
 	// Put the canvas inside the container.
 	blurhashContainer.appendChild(canvas);
 });

web/source/index.js

@@ -60,7 +60,7 @@ skulk({
 			transform: [
 				["babelify", {
 					global: true,
-					ignore: [/node_modules\/(?!(photoswipe.*))/]
+					ignore: [/node_modules\/(?!(.*photoswipe.*))/]
 				}]
 			],
 		},

web/template/status_attachment.tmpl

@@ -29,6 +29,10 @@
     height="{{- .Meta.Small.Height -}}"
     data-blurhash-hash="{{- .Blurhash -}}"
     data-sensitive="{{- .Sensitive -}}"
+    {{- if or (ne .Meta.Focus.X 0.0) (ne .Meta.Focus.Y 0.0) }}
+    data-object-position="{{ objectPosition .Meta.Focus.X .Meta.Focus.Y }}"
+    {{- else }}
+    {{- end }}
 />
 {{- else }}
 <img
@@ -69,6 +73,10 @@
                 data-blurhash-height="{{- .Item.Meta.Small.Height -}}"
                 data-blurhash-hash="{{- .Item.Blurhash -}}"
                 data-blurhash-aspect="{{- .Item.Meta.Small.Aspect -}}"
+                {{- if or (ne .Item.Meta.Focus.X 0.0) (ne .Item.Meta.Focus.Y 0.0) }}
+                data-blurhash-object-position="{{ objectPosition .Item.Meta.Focus.X .Item.Meta.Focus.Y }}"
+                {{- else }}
+                {{- end }}
             ></div>
             {{- end }}
         </summary>