SuperSeriousBusiness
/
GoToSocial
mirror of https://github.com/superseriousbusiness/gotosocial
1
1
Fork 0
Code Issues Projects Releases Wiki Activity

first implementation of search feature

This commit is contained in:
tsmethurst 2021-05-29 19:36:54 +02:00
parent cb54324430
commit 0fe853b1ee
20 changed files with 763 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
PROGRESS.md
View File

@ -100,7 +100,7 @@
* [ ] Timelines
* [ ] /api/v1/timelines/public GET (See the public/federated timeline)
* [ ] /api/v1/timelines/tag/:hashtag GET (Get public statuses that use hashtag)
* [ ] /api/v1/timelines/home GET (View statuses from followed users)
* [x] /api/v1/timelines/home GET (View statuses from followed users)
* [ ] /api/v1/timelines/list/:list_id GET (Get statuses in given list)
* [ ] Conversations
* [ ] /api/v1/conversations GET (Get a list of direct message convos)
@ -121,8 +121,8 @@
* [ ] Streaming
* [ ] /api/v1/streaming WEBSOCKETS (Stream live events to user via websockets)
* [ ] Notifications
* [ ] /api/v1/notifications GET (Get list of notifications)
* [ ] /api/v1/notifications/:id GET (Get a single notification)
* [x] /api/v1/notifications GET (Get list of notifications)
* [x] /api/v1/notifications/:id GET (Get a single notification)
* [ ] /api/v1/notifications/clear POST (Clear all notifications)
* [ ] /api/v1/notifications/:id POST (Clear a single notification)
* [ ] Push
@ -130,8 +130,8 @@
* [ ] /api/v1/push/subscription GET (Get current subscription)
* [ ] /api/v1/push/subscription PUT (Change notification types)
* [ ] /api/v1/push/subscription DELETE (Delete current subscription)
* [ ] Search
* [ ] /api/v2/search GET (Get search query results)
* [x] Search
* [x] /api/v2/search GET (Get search query results)
* [ ] Instance
* [x] /api/v1/instance GET (Get instance information)
* [ ] /api/v1/instance PATCH (Update instance information)
@ -174,7 +174,7 @@
* [ ] Federation modes
* [ ] 'Slow' federation
* [ ] Reputation scoring system for instances
* [ ] 'Greedy' federation
* [x] 'Greedy' federation
* [ ] No federation (insulate this instance from the Fediverse)
* [ ] Allowlist
* [x] Secure HTTP signatures (creation and validation)

88
internal/api/client/search/search.go Normal file
View File

@ -0,0 +1,88 @@
/*
GoToSocial
Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package search
import (
"net/http"
"github.com/sirupsen/logrus"
"github.com/superseriousbusiness/gotosocial/internal/api"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/message"
"github.com/superseriousbusiness/gotosocial/internal/router"
)
const (
// BasePath is the base path for serving v1 of the search API
BasePathV1 = "/api/v1/search"
// BasePathV2 is the base path for serving v2 of the search API
BasePathV2 = "/api/v2/search"
// AccountIDKey -- If provided, statuses returned will be authored only by this account
AccountIDKey = "account_id"
// MaxIDKey -- Return results older than this id
MaxIDKey = "max_id"
// MinIDKey -- Return results immediately newer than this id
MinIDKey = "min_id"
// TypeKey -- Enum(accounts, hashtags, statuses)
TypeKey = "type"
// ExcludeUnreviewedKey -- Filter out unreviewed tags? Defaults to false. Use true when trying to find trending tags.
ExcludeUnreviewedKey = "exclude_unreviewed"
// QueryKey -- The search query
QueryKey = "q"
// ResolveKey -- Attempt WebFinger lookup. Defaults to false.
ResolveKey = "resolve"
// LimitKey -- Maximum number of results to load, per type. Defaults to 20. Max 40.
LimitKey = "limit"
// OffsetKey -- Offset in search results. Used for pagination. Defaults to 0.
OffsetKey = "offset"
// FollowingKey -- Only include accounts that the user is following. Defaults to false.
FollowingKey = "following"
// TypeAccounts --
TypeAccounts = "accounts"
// TypeHashtags --
TypeHashtags = "hashtags"
// TypeStatuses --
TypeStatuses = "statuses"
)
// Module implements the ClientAPIModule interface for everything related to searching
type Module struct {
config *config.Config
processor message.Processor
log *logrus.Logger
}
// New returns a new search module
func New(config *config.Config, processor message.Processor, log *logrus.Logger) api.ClientModule {
return &Module{
config: config,
processor: processor,
log: log,
}
}
// Route attaches all routes from this module to the given router
func (m *Module) Route(r router.Router) error {
r.AttachHandler(http.MethodGet, BasePathV1, m.SearchGETHandler)
r.AttachHandler(http.MethodGet, BasePathV2, m.SearchGETHandler)
return nil
}

151
internal/api/client/search/searchget.go Normal file
View File

@ -0,0 +1,151 @@
/*
GoToSocial
Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package search
import (
"fmt"
"net/http"
"strconv"
"github.com/gin-gonic/gin"
"github.com/sirupsen/logrus"
"github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
)
// SearchGETHandler handles searches for local and remote accounts, statuses, and hashtags.
// It corresponds to the mastodon endpoint described here: https://docs.joinmastodon.org/methods/search/
func (m *Module) SearchGETHandler(c *gin.Context) {
l := m.log.WithFields(logrus.Fields{
"func": "SearchGETHandler",
"request_uri": c.Request.RequestURI,
"user_agent": c.Request.UserAgent(),
"origin_ip": c.ClientIP(),
})
l.Debugf("entering function")
authed, err := oauth.Authed(c, true, true, true, true) // we don't really need an app here but we want everything else
if err != nil {
l.Errorf("error authing search request: %s", err)
c.JSON(http.StatusBadRequest, gin.H{"error": "not authed"})
return
}
accountID := c.Query(AccountIDKey)
maxID := c.Query(MaxIDKey)
minID := c.Query(MinIDKey)
searchType := c.Query(TypeKey)
excludeUnreviewed := false
excludeUnreviewedString := c.Query(ExcludeUnreviewedKey)
if excludeUnreviewedString != "" {
var err error
excludeUnreviewed, err = strconv.ParseBool(excludeUnreviewedString)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("couldn't parse param %s: %s", excludeUnreviewedString, err)})
return
}
}
query := c.Query(QueryKey)
if query == "" {
c.JSON(http.StatusBadRequest, gin.H{"error": "query parameter q was empty"})
return
}
resolve := false
resolveString := c.Query(ResolveKey)
if resolveString != "" {
var err error
resolve, err = strconv.ParseBool(resolveString)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("couldn't parse param %s: %s", resolveString, err)})
return
}
}
limit := 20
limitString := c.Query(LimitKey)
if limitString != "" {
i, err := strconv.ParseInt(limitString, 10, 64)
if err != nil {
l.Debugf("error parsing limit string: %s", err)
c.JSON(http.StatusBadRequest, gin.H{"error": "couldn't parse limit query param"})
return
}
limit = int(i)
}
if limit > 40 {
limit = 40
}
if limit < 1 {
limit = 1
}
offset := 0
offsetString := c.Query(OffsetKey)
if offsetString != "" {
i, err := strconv.ParseInt(offsetString, 10, 64)
if err != nil {
l.Debugf("error parsing offset string: %s", err)
c.JSON(http.StatusBadRequest, gin.H{"error": "couldn't parse offset query param"})
return
}
offset = int(i)
}
if limit > 40 {
limit = 40
}
if limit < 1 {
limit = 1
}
following := false
followingString := c.Query(FollowingKey)
if followingString != "" {
var err error
following, err = strconv.ParseBool(followingString)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("couldn't parse param %s: %s", followingString, err)})
return
}
}
searchQuery := &model.SearchQuery{
AccountID: accountID,
MaxID: maxID,
MinID: minID,
Type: searchType,
ExcludeUnreviewed: excludeUnreviewed,
Query: query,
Resolve: resolve,
Limit: limit,
Offset: offset,
Following: following,
}
results, errWithCode := m.processor.SearchGet(authed, searchQuery)
if errWithCode != nil {
l.Debugf("error searching: %s", errWithCode.Error())
c.JSON(errWithCode.Code(), gin.H{"error": errWithCode.Safe()})
return
}
c.JSON(http.StatusOK, results)
}

52
internal/api/model/search.go Normal file
View File

@ -0,0 +1,52 @@
/*
GoToSocial
Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package model
// SearchQuery corresponds to search parameters as submitted through the client API.
// See https://docs.joinmastodon.org/methods/search/
type SearchQuery struct {
// If provided, statuses returned will be authored only by this account
AccountID string
// Return results older than this id
MaxID string
// Return results immediately newer than this id
MinID string
// Enum(accounts, hashtags, statuses)
Type string
// Filter out unreviewed tags? Defaults to false. Use true when trying to find trending tags.
ExcludeUnreviewed bool
// The search query
Query string
// Attempt WebFinger lookup. Defaults to false.
Resolve bool
// Maximum number of results to load, per type. Defaults to 20. Max 40.
Limit int
// Offset in search results. Used for pagination. Defaults to 0.
Offset int
// Only include accounts that the user is following. Defaults to false.
Following bool
}
// SearchResult corresponds to a search result, containing accounts, statuses, and hashtags.
// See https://docs.joinmastodon.org/methods/search/
type SearchResult struct {
Accounts []Account `json:"accounts"`
Statuses []Status `json:"statuses"`
Hashtags []Tag `json:"hashtags"`
}

5
internal/db/db.go
View File

@ -45,8 +45,9 @@ func (e ErrAlreadyExists) Error() string {
}
type Where struct {
Key string
Value interface{}
Key string
Value interface{}
CaseInsensitive bool
}
// DB provides methods for interacting with an underlying database or other storage mechanism (for now, just postgres).

7
internal/db/pg/pg.go
View File

@ -223,7 +223,12 @@ func (ps *postgresService) GetWhere(where []db.Where, i interface{}) error {
q := ps.conn.Model(i)
for _, w := range where {
q = q.Where("? = ?", pg.Safe(w.Key), w.Value)
if w.CaseInsensitive {
q = q.Where("LOWER(?) = LOWER(?)", pg.Safe(w.Key), w.Value)
} else {
q = q.Where("? = ?", pg.Safe(w.Key), w.Value)
}
}
if err := q.Select(); err != nil {

7
internal/federation/commonbehavior.go
View File

@ -25,6 +25,7 @@ import (
"net/url"
"github.com/go-fed/activity/pub"
"github.com/go-fed/activity/streams"
"github.com/go-fed/activity/streams/vocab"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/util"
@ -59,7 +60,7 @@ import (
func (f *federator) AuthenticateGetInbox(ctx context.Context, w http.ResponseWriter, r *http.Request) (context.Context, bool, error) {
// IMPLEMENTATION NOTE: For GoToSocial, we serve GETS to outboxes and inboxes through
// the CLIENT API, not through the federation API, so we just do nothing here.
return nil, false, nil
return ctx, false, nil
}
// AuthenticateGetOutbox delegates the authentication of a GET to an
@ -84,7 +85,7 @@ func (f *federator) AuthenticateGetInbox(ctx context.Context, w http.ResponseWri
func (f *federator) AuthenticateGetOutbox(ctx context.Context, w http.ResponseWriter, r *http.Request) (context.Context, bool, error) {
// IMPLEMENTATION NOTE: For GoToSocial, we serve GETS to outboxes and inboxes through
// the CLIENT API, not through the federation API, so we just do nothing here.
return nil, false, nil
return ctx, false, nil
}
// GetOutbox returns the OrderedCollection inbox of the actor for this
@ -98,7 +99,7 @@ func (f *federator) AuthenticateGetOutbox(ctx context.Context, w http.ResponseWr
func (f *federator) GetOutbox(ctx context.Context, r *http.Request) (vocab.ActivityStreamsOrderedCollectionPage, error) {
// IMPLEMENTATION NOTE: For GoToSocial, we serve GETS to outboxes and inboxes through
// the CLIENT API, not through the federation API, so we just do nothing here.
return nil, nil
return streams.NewActivityStreamsOrderedCollectionPage(), nil
}
// NewTransport returns a new Transport on behalf of a specific actor.

14
internal/federation/federatingdb/followers.go
View File

@ -10,6 +10,7 @@ import (
"github.com/sirupsen/logrus"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/util"
)
// Followers obtains the Followers Collection for an actor with the
@ -28,8 +29,17 @@ func (f *federatingDB) Followers(c context.Context, actorIRI *url.URL) (follower
l.Debugf("entering FOLLOWERS function with actorIRI %s", actorIRI.String())
acct := &gtsmodel.Account{}
if err := f.db.GetWhere([]db.Where{{Key: "uri", Value: actorIRI.String()}}, acct); err != nil {
return nil, fmt.Errorf("db error getting account with uri %s: %s", actorIRI.String(), err)
if util.IsUserPath(actorIRI) {
if err := f.db.GetWhere([]db.Where{{Key: "uri", Value: actorIRI.String()}}, acct); err != nil {
return nil, fmt.Errorf("db error getting account with uri %s: %s", actorIRI.String(), err)
}
} else if util.IsFollowersPath(actorIRI) {
if err := f.db.GetWhere([]db.Where{{Key: "followers_uri", Value: actorIRI.String()}}, acct); err != nil {
return nil, fmt.Errorf("db error getting account with followers uri %s: %s", actorIRI.String(), err)
}
} else {
return nil, fmt.Errorf("could not parse actor IRI %s as users or followers path", actorIRI.String())
}
acctFollowers := []gtsmodel.Follow{}

13
internal/federation/federatingdb/following.go
View File

@ -10,6 +10,7 @@ import (
"github.com/sirupsen/logrus"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/util"
)
// Following obtains the Following Collection for an actor with the
@ -28,8 +29,16 @@ func (f *federatingDB) Following(c context.Context, actorIRI *url.URL) (followin
l.Debugf("entering FOLLOWING function with actorIRI %s", actorIRI.String())
acct := &gtsmodel.Account{}
if err := f.db.GetWhere([]db.Where{{Key: "uri", Value: actorIRI.String()}}, acct); err != nil {
return nil, fmt.Errorf("db error getting account with uri %s: %s", actorIRI.String(), err)
if util.IsUserPath(actorIRI) {
if err := f.db.GetWhere([]db.Where{{Key: "uri", Value: actorIRI.String()}}, acct); err != nil {
return nil, fmt.Errorf("db error getting account with uri %s: %s", actorIRI.String(), err)
}
} else if util.IsFollowingPath(actorIRI) {
if err := f.db.GetWhere([]db.Where{{Key: "following_uri", Value: actorIRI.String()}}, acct); err != nil {
return nil, fmt.Errorf("db error getting account with following uri %s: %s", actorIRI.String(), err)
}
} else {
return nil, fmt.Errorf("could not parse actor IRI %s as users or following path", actorIRI.String())
}
acctFollowing := []gtsmodel.Follow{}

7
internal/federation/federatingdb/lock.go
View File

@ -42,6 +42,10 @@ func (f *federatingDB) Lock(c context.Context, id *url.URL) error {
// Strategy: create a new lock, if stored, continue. Otherwise, lock the
// existing mutex.
if id == nil {
return errors.New("Lock: id was nil")
}
mu := &sync.Mutex{}
mu.Lock() // Optimistically lock if we do store it.
i, loaded := f.locks.LoadOrStore(id.String(), mu)
@ -59,6 +63,9 @@ func (f *federatingDB) Lock(c context.Context, id *url.URL) error {
func (f *federatingDB) Unlock(c context.Context, id *url.URL) error {
// Once Go-Fed is done calling Database methods, the relevant `id`
// entries are unlocked.
if id == nil {
return errors.New("Unlock: id was nil")
}
i, ok := f.locks.Load(id.String())
if !ok {

5
internal/federation/federatingprotocol.go
View File

@ -26,6 +26,7 @@ import (
"net/url"
"github.com/go-fed/activity/pub"
"github.com/go-fed/activity/streams"
"github.com/go-fed/activity/streams/vocab"
"github.com/sirupsen/logrus"
"github.com/superseriousbusiness/gotosocial/internal/db"
@ -310,7 +311,7 @@ func (f *federator) MaxDeliveryRecursionDepth(ctx context.Context) int {
// logic to be used, but the implementation must not modify it.
func (f *federator) FilterForwarding(ctx context.Context, potentialRecipients []*url.URL, a pub.Activity) ([]*url.URL, error) {
// TODO
return nil, nil
return []*url.URL{}, nil
}
// GetInbox returns the OrderedCollection inbox of the actor for this
@ -324,5 +325,5 @@ func (f *federator) FilterForwarding(ctx context.Context, potentialRecipients []
func (f *federator) GetInbox(ctx context.Context, r *http.Request) (vocab.ActivityStreamsOrderedCollectionPage, error) {
// IMPLEMENTATION NOTE: For GoToSocial, we serve GETS to outboxes and inboxes through
// the CLIENT API, not through the federation API, so we just do nothing here.
return nil, nil
return streams.NewActivityStreamsOrderedCollectionPage(), nil
}

3
internal/federation/federator.go
View File

@ -40,6 +40,9 @@ type Federator interface {
// AuthenticateFederatedRequest can be used to check the authenticity of incoming http-signed requests for federating resources.
// The given username will be used to create a transport for making outgoing requests. See the implementation for more detailed comments.
AuthenticateFederatedRequest(username string, r *http.Request) (*url.URL, error)
// FingerRemoteAccount performs a webfinger lookup for a remote account, using the .well-known path. It will return the ActivityPub URI for that
// account, or an error if it doesn't exist or can't be retrieved.
FingerRemoteAccount(requestingUsername string, targetUsername string, targetDomain string) (*url.URL, error)
// DereferenceRemoteAccount can be used to get the representation of a remote account, based on the account ID (which is a URI).
// The given username will be used to create a transport for making outgoing requests. See the implementation for more detailed comments.
DereferenceRemoteAccount(username string, remoteAccountID *url.URL) (typeutils.Accountable, error)

69
internal/federation/finger.go Normal file
View File

@ -0,0 +1,69 @@
/*
GoToSocial
Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package federation
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/url"
"strings"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
)
func (f *federator) FingerRemoteAccount(requestingUsername string, targetUsername string, targetDomain string) (*url.URL, error) {
t, err := f.GetTransportForUser(requestingUsername)
if err != nil {
return nil, fmt.Errorf("FingerRemoteAccount: error getting transport for username %s while dereferencing @%s@%s: %s", requestingUsername, targetUsername, targetDomain, err)
}
b, err := t.Finger(context.Background(), targetUsername, targetDomain)
if err != nil {
return nil, fmt.Errorf("FingerRemoteAccount: error doing request on behalf of username %s while dereferencing @%s@%s: %s", requestingUsername, targetUsername, targetDomain, err)
}
resp := &apimodel.WebfingerAccountResponse{}
if err := json.Unmarshal(b, resp); err != nil {
return nil, fmt.Errorf("FingerRemoteAccount: could not unmarshal server response as WebfingerAccountResponse on behalf of username %s while dereferencing @%s@%s: %s", requestingUsername, targetUsername, targetDomain, err)
}
if len(resp.Links) == 0 {
return nil, fmt.Errorf("FingerRemoteAccount: no links found in webfinger response %s", string(b))
}
// look through the links for the first one that matches "application/activity+json", this is what we need
for _, l := range resp.Links {
if strings.EqualFold(l.Type, "application/activity+json") {
if l.Href == "" || l.Rel != "self" {
continue
}
accountURI, err := url.Parse(l.Href)
if err != nil {
return nil, fmt.Errorf("FingerRemoteAccount: couldn't parse url %s: %s", l.Href, err)
}
// found it!
return accountURI, nil
}
}
return nil, errors.New("FingerRemoteAccount: no match found in webfinger response")
}

3
internal/gotosocial/actions.go
View File

@ -38,6 +38,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/api/client/instance"
mediaModule "github.com/superseriousbusiness/gotosocial/internal/api/client/media"
"github.com/superseriousbusiness/gotosocial/internal/api/client/notification"
"github.com/superseriousbusiness/gotosocial/internal/api/client/search"
"github.com/superseriousbusiness/gotosocial/internal/api/client/status"
"github.com/superseriousbusiness/gotosocial/internal/api/client/timeline"
"github.com/superseriousbusiness/gotosocial/internal/api/s2s/user"
@ -122,6 +123,7 @@ var Run action.GTSAction = func(ctx context.Context, c *config.Config, log *logr
usersModule := user.New(c, processor, log)
timelineModule := timeline.New(c, processor, log)
notificationModule := notification.New(c, processor, log)
searchModule := search.New(c, processor, log)
mm := mediaModule.New(c, processor, log)
fileServerModule := fileserver.New(c, processor, log)
adminModule := admin.New(c, processor, log)
@ -146,6 +148,7 @@ var Run action.GTSAction = func(ctx context.Context, c *config.Config, log *logr
usersModule,
timelineModule,
notificationModule,
searchModule,
}
for _, m := range apis {

4
internal/message/fromfederatorprocess.go
View File

@ -105,7 +105,9 @@ func (p *processor) processFromFederator(federatorMsg gtsmodel.FromFederator) er
}
if err := p.db.Put(incomingAnnounce); err != nil {
return fmt.Errorf("error adding dereferenced announce to the db: %s", err)
if _, ok := err.(db.ErrAlreadyExists); !ok {
return fmt.Errorf("error adding dereferenced announce to the db: %s", err)
}
}
if err := p.notifyAnnounce(incomingAnnounce); err != nil {

3
internal/message/processor.go
View File

@ -109,6 +109,9 @@ type Processor interface {
// NotificationsGet
NotificationsGet(authed *oauth.Auth, limit int, maxID string) ([]*apimodel.Notification, ErrorWithCode)
// SearchGet performs a search with the given params, resolving/dereferencing remotely as desired
SearchGet(authed *oauth.Auth, searchQuery *apimodel.SearchQuery) (*apimodel.SearchResult, ErrorWithCode)
// StatusCreate processes the given form to create a new status, returning the api model representation of that status if it's OK.
StatusCreate(authed *oauth.Auth, form *apimodel.AdvancedStatusCreateForm) (*apimodel.Status, error)
// StatusDelete processes the delete of a given status, returning the deleted status if the delete goes through.

292
internal/message/searchprocess.go Normal file
View File

@ -0,0 +1,292 @@
/*
GoToSocial
Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package message
import (
"errors"
"net/url"
"strings"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/util"
)
func (p *processor) SearchGet(authed *oauth.Auth, searchQuery *apimodel.SearchQuery) (*apimodel.SearchResult, ErrorWithCode) {
results := &apimodel.SearchResult{
Accounts: []apimodel.Account{},
Statuses: []apimodel.Status{},
Hashtags: []apimodel.Tag{},
}
foundAccounts := []*gtsmodel.Account{}
foundStatuses := []*gtsmodel.Status{}
// foundHashtags := []*gtsmodel.Tag{}
// convert the query to lowercase and trim leading/trailing spaces
query := strings.ToLower(strings.TrimSpace(searchQuery.Query))
// check if the query is a URI and just do a lookup for that, straight up
if uri, err := url.Parse(query); err == nil {
// 1. check if it's a status
foundStatus, err := p.searchStatusByURI(authed, uri, searchQuery.Resolve)
if err != nil {
return nil, NewErrorInternalError(err)
}
if foundStatus != nil {
foundStatuses = append(foundStatuses, foundStatus)
}
// 2. check if it's an account
foundAccount, err := p.searchAccountByURI(authed, uri, searchQuery.Resolve)
if err != nil {
return nil, NewErrorInternalError(err)
}
if foundAccount != nil {
foundAccounts = append(foundAccounts, foundAccount)
}
}
// check if the query is something like @whatever_username@example.org -- this means it's a remote account
if util.IsMention(searchQuery.Query) {
foundAccount, err := p.searchAccountByMention(authed, searchQuery.Query, searchQuery.Resolve)
if err != nil {
return nil, NewErrorInternalError(err)
}
if foundAccount != nil {
foundAccounts = append(foundAccounts, foundAccount)
}
}
/*
FROM HERE ON we have our search results, it's just a matter of filtering them according to what this user is allowed to see,
and then converting them into our frontend format.
*/
for _, foundAccount := range foundAccounts {
// make sure there's no block in either direction between the account and the requester
if blocked, err := p.db.Blocked(authed.Account.ID, foundAccount.ID); err == nil && !blocked {
// all good, convert it and add it to the results
acctMasto, err := p.tc.AccountToMastoPublic(foundAccount)
if err != nil {
return nil, NewErrorInternalError(err)
}
results.Accounts = append(results.Accounts, *acctMasto)
}
}
for _, foundStatus := range foundStatuses {
statusOwner := &gtsmodel.Account{}
if err := p.db.GetByID(foundStatus.AccountID, statusOwner); err != nil {
continue
}
relevantAccounts, err := p.db.PullRelevantAccountsFromStatus(foundStatus)
if err != nil {
continue
}
if visible, err := p.db.StatusVisible(foundStatus, statusOwner, authed.Account, relevantAccounts); !visible || err != nil {
continue
}
statusMasto, err := p.tc.StatusToMasto(foundStatus, statusOwner, authed.Account, relevantAccounts.BoostedAccount, relevantAccounts.ReplyToAccount, nil)
if err != nil {
continue
}
results.Statuses = append(results.Statuses, *statusMasto)
}
return results, nil
}
func (p *processor) searchStatusByURI(authed *oauth.Auth, uri *url.URL, resolve bool) (foundStatus *gtsmodel.Status, err error) {
// 1. check if it's a status
maybeStatus := &gtsmodel.Status{}
if err = p.db.GetWhere([]db.Where{{Key: "uri", Value: uri.String(), CaseInsensitive: true}}, maybeStatus); err == nil {
// we have it and it's a status
foundStatus = maybeStatus
return
} else if err = p.db.GetWhere([]db.Where{{Key: "url", Value: uri.String(), CaseInsensitive: true}}, maybeStatus); err == nil {
// we have it and it's a status
foundStatus = maybeStatus
return
}
// we don't have it locally so dereference it if we're allowed to
if resolve {
statusable, err := p.federator.DereferenceRemoteStatus(authed.Account.Username, uri)
if err == nil {
// it IS a status!
// extract the status owner's IRI from the statusable
var statusOwnerURI *url.URL
statusAttributedTo := statusable.GetActivityStreamsAttributedTo()
for i := statusAttributedTo.Begin(); i != statusAttributedTo.End(); i = i.Next() {
if i.IsIRI() {
statusOwnerURI = i.GetIRI()
break
}
}
if statusOwnerURI == nil {
return nil, NewErrorInternalError(errors.New("couldn't extract ownerAccountURI from statusable"))
}
// make sure the status owner exists in the db by searching for it
_, err := p.searchAccountByURI(authed, statusOwnerURI, resolve)
if err != nil {
return nil, err
}
// we have the status owner, we have the dereferenced status, so now we should finish dereferencing the status properly
// first turn it into a gtsmodel.Status
status, err := p.tc.ASStatusToStatus(statusable)
if err != nil {
return nil, NewErrorInternalError(err)
}
// put it in the DB so it gets a UUID
if err := p.db.Put(status); err != nil {
return nil, NewErrorInternalError(err)
}
// properly dereference everything in the status (media attachments etc)
if err := p.dereferenceStatusFields(status, authed.Account.Username); err != nil {
return nil, NewErrorInternalError(err)
}
// update with the nicely dereferenced status
if err := p.db.UpdateByID(status.ID, status); err != nil {
return nil, NewErrorInternalError(err)
}
foundStatus = status
}
}
return
}
func (p *processor) searchAccountByURI(authed *oauth.Auth, uri *url.URL, resolve bool) (foundAccount *gtsmodel.Account, err error) {
maybeAccount := &gtsmodel.Account{}
if err = p.db.GetWhere([]db.Where{{Key: "uri", Value: uri.String(), CaseInsensitive: true}}, maybeAccount); err == nil {
// we have it and it's an account
foundAccount = maybeAccount
return
} else if err = p.db.GetWhere([]db.Where{{Key: "url", Value: uri.String(), CaseInsensitive: true}}, maybeAccount); err == nil {
// we have it and it's an account
foundAccount = maybeAccount
return
}
if resolve {
// we don't have it locally so try and dereference it
accountable, err := p.federator.DereferenceRemoteAccount(authed.Account.Username, uri)
if err == nil {
// it IS an account!
account, err := p.tc.ASRepresentationToAccount(accountable, false)
if err != nil {
return nil, NewErrorInternalError(err)
}
if err := p.db.Put(account); err != nil {
return nil, NewErrorInternalError(err)
}
if err := p.dereferenceAccountFields(account, authed.Account.Username, false); err != nil {
return nil, NewErrorInternalError(err)
}
foundAccount = account
}
}
return
}
func (p *processor) searchAccountByMention(authed *oauth.Auth, mention string, resolve bool) (foundAccount *gtsmodel.Account, err error) {
// query is for a remote account
username, domain, err := util.ExtractMentionParts(mention)
if err != nil {
return nil, NewErrorBadRequest(err)
}
// if it's a local account we can skip a whole bunch of stuff
maybeAcct := &gtsmodel.Account{}
if domain == p.config.Host {
if p.db.GetLocalAccountByUsername(username, maybeAcct); err != nil {
return
}
foundAccount = maybeAcct
return
}
// it's not a local account so first we'll check if it's in the database already...
where := []db.Where{
{Key: "username", Value: username, CaseInsensitive: true},
{Key: "domain", Value: domain, CaseInsensitive: true},
}
err = p.db.GetWhere(where, maybeAcct)
if err == nil {
// we've got it stored locally already!
foundAccount = maybeAcct
return
}
if _, ok := err.(db.ErrNoEntries); !ok {
// if it's not errNoEntries there's been a real database error so bail at this point
return nil, NewErrorInternalError(err)
}
// we got a db.ErrNoEntries, so we just don't have the account locally stored -- check if we can dereference it
if resolve {
// we're allowed to resolve it so let's try
// first we need to webfinger the remote account to convert the username and domain into the activitypub URI for the account
acctURI, err := p.federator.FingerRemoteAccount(authed.Account.Username, username, domain)
if err != nil {
// something went wrong doing the webfinger lookup so we can't process the request
return nil, NewErrorInternalError(err)
}
// dereference the account based on the URI we retrieved from the webfinger lookup
accountable, err := p.federator.DereferenceRemoteAccount(authed.Account.Username, acctURI)
if err != nil {
// something went wrong doing the dereferencing so we can't process the request
return nil, NewErrorInternalError(err)
}
// convert the dereferenced account to the gts model of that account
foundAccount, err = p.tc.ASRepresentationToAccount(accountable, false)
if err != nil {
// something went wrong doing the conversion to a gtsmodel.Account so we can't process the request
return nil, NewErrorInternalError(err)
}
// put this new account in our database
if err := p.db.Put(foundAccount); err != nil {
return nil, NewErrorInternalError(err)
}
// properly dereference all the fields on the account immediately
if err := p.dereferenceAccountFields(foundAccount, authed.Account.Username, true); err != nil {
return nil, NewErrorInternalError(err)
}
}
return
}

41
internal/transport/transport.go
View File

@ -19,6 +19,8 @@ import (
type Transport interface {
pub.Transport
DereferenceMedia(c context.Context, iri *url.URL, expectedContentType string) ([]byte, error)
// Finger performs a webfinger request with the given username and domain, and returns the bytes from the response body.
Finger(c context.Context, targetUsername string, targetDomains string) ([]byte, error)
}
// transport implements the Transport interface
@ -83,3 +85,42 @@ func (t *transport) DereferenceMedia(c context.Context, iri *url.URL, expectedCo
}
return ioutil.ReadAll(resp.Body)
}
func (t *transport) Finger(c context.Context, targetUsername string, targetDomain string) ([]byte, error) {
l := t.log.WithField("func", "Finger")
urlString := fmt.Sprintf("https://%s/.well-known/webfinger?resource=acct:%s@%s", targetDomain, targetUsername, targetDomain)
l.Debugf("performing GET to %s", urlString)
iri, err := url.Parse(urlString)
if err != nil {
return nil, fmt.Errorf("Finger: error parsing url %s: %s", urlString, err)
}
l.Debugf("performing GET to %s", iri.String())
req, err := http.NewRequest("GET", iri.String(), nil)
if err != nil {
return nil, err
}
req = req.WithContext(c)
req.Header.Add("Accept", "application/json")
req.Header.Add("Accept", "application/jrd+json")
req.Header.Add("Date", t.clock.Now().UTC().Format("Mon, 02 Jan 2006 15:04:05")+" GMT")
req.Header.Add("User-Agent", fmt.Sprintf("%s %s", t.appAgent, t.gofedAgent))
req.Header.Set("Host", iri.Host)
t.getSignerMu.Lock()
err = t.getSigner.SignRequest(t.privkey, t.pubKeyID, req, nil)
t.getSignerMu.Unlock()
if err != nil {
return nil, err
}
resp, err := t.client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("GET request to %s failed (%d): %s", iri.String(), resp.StatusCode, resp.Status)
}
return ioutil.ReadAll(resp.Body)
}

1
internal/typeutils/internaltoas.go
View File

@ -252,6 +252,7 @@ func (c *converter) AccountToAS(a *gtsmodel.Account) (vocab.ActivityStreamsPerso
headerImage.SetActivityStreamsUrl(headerURLProperty)
headerProperty.AppendActivityStreamsImage(headerImage)
person.SetActivityStreamsImage(headerProperty)
}
return person, nil

5
internal/util/statustools.go
View File

@ -77,6 +77,11 @@ func ExtractMentionParts(mention string) (username, domain string, err error) {
return
}
// IsMention returns true if the passed string looks like @whatever@example.org
func IsMention(mention string) bool {
return mentionNameRegex.MatchString(strings.ToLower(mention))
}
// unique returns a deduplicated version of a given string slice.
func unique(s []string) []string {
keys := make(map[string]bool)