Support filter tags from instances.jpt

README.md

@@ -3,7 +3,6 @@
 ## Dev
 
 ```terminal
-$ git submodule update --init --recursive
 $ yarn install --pure-lockfile
 ```
 

package.json

@@ -33,6 +33,7 @@
     "js-yaml": "^4.1.0",
     "lodash": "^4.17.15",
     "meilisearch": "^0.35.0",
+    "memoizee": "^0.4.17",
     "mkdirp": "^1.0.4",
     "morgan": "^1.9.1",
     "multer": "^1.4.5-lts.1",
@@ -53,6 +54,7 @@
     "@types/fluent-ffmpeg": "^2.1.20",
     "@types/fs-extra": "^9.0.13",
     "@types/lodash": "^4.14.182",
+    "@types/memoizee": "^0.4.11",
     "@types/mkdirp": "^1.0.2",
     "@types/morgan": "^1.9.3",
     "@types/multer": "^1.4.7",

server/lib/controllers/searcher.ts

@@ -1,6 +1,7 @@
 import { ResultList } from '@peertube/peertube-types'
 import { CONFIG } from '../../initializers/constants'
 import { CommonSearch } from '../../types/search-query/common-search.model'
+import { listFilteredInstanceHostsMemoized } from '../requests/instances-index'
 
 export class Searcher <T extends CommonSearch, R, F> {
 
@@ -20,6 +21,19 @@ export class Searcher <T extends CommonSearch, R, F> {
       query.blockedHosts = query.blockedHosts.concat(CONFIG.API.BLACKLIST.HOSTS)
     }
 
+    if (queryArg.filterTag) {
+      if (!Array.isArray(query.hosts)) query.hosts = []
+
+      query.hosts = query.hosts.concat(await listFilteredInstanceHostsMemoized(query.filterTag))
+
+      if (query.hosts.length === 0) {
+        return {
+          total: 0,
+          data: []
+        }
+      }
+    }
+
     const resultList = await this.queryFn(query)
 
     return {

server/lib/meilisearch/meilisearch-channels.ts

@@ -25,6 +25,10 @@ export async function queryChannels (search: ChannelsSearchQuery) {
     filter.push(`handle IN ${buildInValuesArray(search.handles)}`)
   }
 
+  if (search.hosts && search.hosts.length !== 0) {
+    filter.push(`host IN ${buildInValuesArray(search.hosts)}`)
+  }
+
   if (search.blockedHosts && search.blockedHosts.length !== 0) {
     filter.push(`host NOT IN ${buildInValuesArray(search.blockedHosts)}`)
   }

server/lib/meilisearch/meilisearch-playlists.ts

@@ -13,7 +13,7 @@ export async function queryPlaylists (search: PlaylistsSearchQuery) {
     'videosLength != 0'
   ]
 
-  if (search.host) filter.push(`ownerAccount.host = '${search.host}'`)
+  if (search.host) filter.push(`host = '${search.host}'`)
 
   if (search.blockedAccounts && search.blockedAccounts.length !== 0) {
     filter.push(`ownerAccount.handle NOT IN ${buildInValuesArray(search.blockedAccounts)}`)
@@ -23,6 +23,10 @@ export async function queryPlaylists (search: PlaylistsSearchQuery) {
     filter.push(`host NOT IN ${buildInValuesArray(search.blockedHosts)}`)
   }
 
+  if (search.hosts && search.hosts.length !== 0) {
+    filter.push(`host IN ${buildInValuesArray(search.hosts)}`)
+  }
+
   if (search.uuids) addUUIDFilters(filter, search.uuids)
 
   logger.debug({ filter }, 'Will query Meilisearch for playlists.')

server/lib/meilisearch/meilisearch-videos.ts

@@ -48,7 +48,7 @@ export async function queryVideos (search: VideosSearchQuery) {
   if (search.nsfw && search.nsfw !== 'both') filter.push(`nsfw = ${search.nsfw}`)
 
   if (exists(search.isLive)) filter.push(`isLive = ${search.isLive}`)
-  if (search.host) filter.push(`account.host = '${search.host}'`)
+  if (search.host) filter.push(`host = '${search.host}'`)
   if (search.uuids) addUUIDFilters(filter, search.uuids)
 
   if (search.tagsOneOf && search.tagsOneOf.length !== 0) {
@@ -63,6 +63,10 @@ export async function queryVideos (search: VideosSearchQuery) {
     filter.push(clause)
   }
 
+  if (search.hosts && search.hosts.length !== 0) {
+    filter.push(`host IN ${buildInValuesArray(search.hosts)}`)
+  }
+
   logger.debug({ filter }, 'Will query Meilisearch for videos.')
 
   if (search.boostLanguages && search.boostLanguages.length !== 0) {

server/lib/requests/instances-index.ts

@@ -1,5 +1,7 @@
 import { CONFIG } from '../../initializers/constants'
 import { doRequest } from '../../helpers/requests'
+import memoizee from 'memoizee'
+import { logger } from '../../helpers/logger'
 
 export async function listIndexInstancesHost (): Promise<string[]> {
   const uri = CONFIG.INSTANCES_INDEX.URL
@@ -29,3 +31,21 @@ export async function getMajorInstanceVersion (host: string): Promise<number> {
     return 0
   }
 }
+
+export const listFilteredInstanceHostsMemoized = memoizee(async (filterTag: string) => {
+  const uri = CONFIG.INSTANCES_INDEX.URL
+
+  const qs = {
+    healthy: true,
+    count: 5000,
+    filterTag
+  }
+
+  const { body } = await doRequest<any>({ uri, qs, json: true })
+
+  const hosts = body.data.map(o => o.host as string)
+
+  logger.info({ hosts }, 'Getting filtered instance hosts for tag ' + filterTag)
+
+  return hosts
+}, { maxAge: 60000 }) // 1 minute

server/middlewares/validators/search.ts

@@ -14,6 +14,8 @@ const commonFiltersValidators = [
     .optional()
     .customSanitizer(toArray)
     .custom(isStringArray).withMessage('Should have a valid hosts array'),
+  check('filterTag')
+    .optional(),
 
   (req: express.Request, res: express.Response, next: express.NextFunction) => {
     logger.debug({ query: req.query, body: req.body }, 'Checking commons filters query')

server/types/search-query/common-search.model.ts

@@ -2,4 +2,6 @@ export type CommonSearch = {
   blockedAccounts?: string[]
   blockedHosts?: string[]
   fromHost?: string
+  hosts?: string[]
+  filterTag?: string
 }

yarn.lock

@@ -1311,6 +1311,11 @@
   resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.191.tgz#09511e7f7cba275acd8b419ddac8da9a6a79e2fa"
   integrity sha512-BdZ5BCCvho3EIXw6wUCXHe7rS53AIDPLE+JzwgT+OsJk53oBfbSmZZ7CX4VaRoN78N+TJpFi9QPlfIVNmJYWxQ==
 
+"@types/memoizee@^0.4.11":
+  version "0.4.11"
+  resolved "https://registry.yarnpkg.com/@types/memoizee/-/memoizee-0.4.11.tgz#da8897f0064bff3e845b5185e2a323bbd5c8d0a3"
+  integrity sha512-2gyorIBZu8GoDr9pYjROkxWWcFtHCquF7TVbN2I+/OvgZhnIGQS0vX5KJz4lXNKb8XOSfxFOSG5OLru1ESqLUg==
+
 "@types/memoizee@^0.4.2":
   version "0.4.8"
   resolved "https://registry.yarnpkg.com/@types/memoizee/-/memoizee-0.4.8.tgz#04adc0c266a0f5d72db0556fdda2ba17dad9b519"
@@ -2026,6 +2031,14 @@ d@1, d@^1.0.1:
     es5-ext "^0.10.50"
     type "^1.0.1"
 
+d@^1.0.2:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/d/-/d-1.0.2.tgz#2aefd554b81981e7dccf72d6842ae725cb17e5de"
+  integrity sha512-MOqHvMWF9/9MX6nza0KgvFH4HpMU0EF5uUDXqX/BtxtU8NfB0QzRtJ8Oe/6SuS4kbhyzVJwjd97EA4PKrzJ8bw==
+  dependencies:
+    es5-ext "^0.10.64"
+    type "^2.7.2"
+
 dashdash@^1.12.0:
   version "1.14.1"
   resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0"
@@ -2242,6 +2255,16 @@ es5-ext@^0.10.35, es5-ext@^0.10.46, es5-ext@^0.10.50, es5-ext@^0.10.53, es5-ext@
     es6-symbol "^3.1.3"
     next-tick "^1.1.0"
 
+es5-ext@^0.10.62, es5-ext@^0.10.64:
+  version "0.10.64"
+  resolved "https://registry.yarnpkg.com/es5-ext/-/es5-ext-0.10.64.tgz#12e4ffb48f1ba2ea777f1fcdd1918ef73ea21714"
+  integrity sha512-p2snDhiLaXe6dahss1LddxqEm+SkuDvV8dnIQG0MWjyHpcMNfXKPE+/Cc0y+PhxJX3A4xGNeFCj5oc0BUh6deg==
+  dependencies:
+    es6-iterator "^2.0.3"
+    es6-symbol "^3.1.3"
+    esniff "^2.0.1"
+    next-tick "^1.1.0"
+
 es6-iterator@^2.0.3:
   version "2.0.3"
   resolved "https://registry.yarnpkg.com/es6-iterator/-/es6-iterator-2.0.3.tgz#a7de889141a05a94b0854403b2d0a0fbfa98f3b7"
@@ -2468,6 +2491,16 @@ eslint@^8.16.0:
     strip-json-comments "^3.1.0"
     text-table "^0.2.0"
 
+esniff@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/esniff/-/esniff-2.0.1.tgz#a4d4b43a5c71c7ec51c51098c1d8a29081f9b308"
+  integrity sha512-kTUIGKQ/mDPFoJ0oVfcmyJn4iBDRptjNVIzwIFR7tqWXdVI9xfA2RMwY/gbSpJG3lkdWNEjLap/NqVHZiJsdfg==
+  dependencies:
+    d "^1.0.1"
+    es5-ext "^0.10.62"
+    event-emitter "^0.3.5"
+    type "^2.7.2"
+
 espree@^9.4.0:
   version "9.4.1"
   resolved "https://registry.yarnpkg.com/espree/-/espree-9.4.1.tgz#51d6092615567a2c2cff7833445e37c28c0065bd"
@@ -3487,6 +3520,20 @@ memoizee@^0.4.14:
     next-tick "^1.1.0"
     timers-ext "^0.1.7"
 
+memoizee@^0.4.17:
+  version "0.4.17"
+  resolved "https://registry.yarnpkg.com/memoizee/-/memoizee-0.4.17.tgz#942a5f8acee281fa6fb9c620bddc57e3b7382949"
+  integrity sha512-DGqD7Hjpi/1or4F/aYAspXKNm5Yili0QDAFAY4QYvpqpgiY6+1jOfqpmByzjxbWd/T9mChbCArXAbDAsTm5oXA==
+  dependencies:
+    d "^1.0.2"
+    es5-ext "^0.10.64"
+    es6-weak-map "^2.0.3"
+    event-emitter "^0.3.5"
+    is-promise "^2.2.2"
+    lru-queue "^0.1.0"
+    next-tick "^1.1.0"
+    timers-ext "^0.1.7"
+
 merge-descriptors@1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/merge-descriptors/-/merge-descriptors-1.0.1.tgz#b00aaa556dd8b44568150ec9d1b953f3f90cbb61"