diff --git a/README.md b/README.md index 7f11eb2..8b01305 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,6 @@ ## Dev ```terminal -$ git submodule update --init --recursive $ yarn install --pure-lockfile ``` diff --git a/package.json b/package.json index 730a044..6d21b07 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "js-yaml": "^4.1.0", "lodash": "^4.17.15", "meilisearch": "^0.35.0", + "memoizee": "^0.4.17", "mkdirp": "^1.0.4", "morgan": "^1.9.1", "multer": "^1.4.5-lts.1", @@ -53,6 +54,7 @@ "@types/fluent-ffmpeg": "^2.1.20", "@types/fs-extra": "^9.0.13", "@types/lodash": "^4.14.182", + "@types/memoizee": "^0.4.11", "@types/mkdirp": "^1.0.2", "@types/morgan": "^1.9.3", "@types/multer": "^1.4.7", diff --git a/server/lib/controllers/searcher.ts b/server/lib/controllers/searcher.ts index 7a75d83..397a00d 100644 --- a/server/lib/controllers/searcher.ts +++ b/server/lib/controllers/searcher.ts @@ -1,6 +1,7 @@ import { ResultList } from '@peertube/peertube-types' import { CONFIG } from '../../initializers/constants' import { CommonSearch } from '../../types/search-query/common-search.model' +import { listFilteredInstanceHostsMemoized } from '../requests/instances-index' export class Searcher { @@ -20,6 +21,19 @@ export class Searcher { query.blockedHosts = query.blockedHosts.concat(CONFIG.API.BLACKLIST.HOSTS) } + if (queryArg.filterTag) { + if (!Array.isArray(query.hosts)) query.hosts = [] + + query.hosts = query.hosts.concat(await listFilteredInstanceHostsMemoized(query.filterTag)) + + if (query.hosts.length === 0) { + return { + total: 0, + data: [] + } + } + } + const resultList = await this.queryFn(query) return { diff --git a/server/lib/meilisearch/meilisearch-channels.ts b/server/lib/meilisearch/meilisearch-channels.ts index 9b78a31..3b8468d 100644 --- a/server/lib/meilisearch/meilisearch-channels.ts +++ b/server/lib/meilisearch/meilisearch-channels.ts @@ -25,6 +25,10 @@ export async function queryChannels (search: ChannelsSearchQuery) { filter.push(`handle IN ${buildInValuesArray(search.handles)}`) } + if (search.hosts && search.hosts.length !== 0) { + filter.push(`host IN ${buildInValuesArray(search.hosts)}`) + } + if (search.blockedHosts && search.blockedHosts.length !== 0) { filter.push(`host NOT IN ${buildInValuesArray(search.blockedHosts)}`) } diff --git a/server/lib/meilisearch/meilisearch-playlists.ts b/server/lib/meilisearch/meilisearch-playlists.ts index 43a6b20..c0ce93b 100644 --- a/server/lib/meilisearch/meilisearch-playlists.ts +++ b/server/lib/meilisearch/meilisearch-playlists.ts @@ -13,7 +13,7 @@ export async function queryPlaylists (search: PlaylistsSearchQuery) { 'videosLength != 0' ] - if (search.host) filter.push(`ownerAccount.host = '${search.host}'`) + if (search.host) filter.push(`host = '${search.host}'`) if (search.blockedAccounts && search.blockedAccounts.length !== 0) { filter.push(`ownerAccount.handle NOT IN ${buildInValuesArray(search.blockedAccounts)}`) @@ -23,6 +23,10 @@ export async function queryPlaylists (search: PlaylistsSearchQuery) { filter.push(`host NOT IN ${buildInValuesArray(search.blockedHosts)}`) } + if (search.hosts && search.hosts.length !== 0) { + filter.push(`host IN ${buildInValuesArray(search.hosts)}`) + } + if (search.uuids) addUUIDFilters(filter, search.uuids) logger.debug({ filter }, 'Will query Meilisearch for playlists.') diff --git a/server/lib/meilisearch/meilisearch-videos.ts b/server/lib/meilisearch/meilisearch-videos.ts index 462d2ff..26c397b 100644 --- a/server/lib/meilisearch/meilisearch-videos.ts +++ b/server/lib/meilisearch/meilisearch-videos.ts @@ -48,7 +48,7 @@ export async function queryVideos (search: VideosSearchQuery) { if (search.nsfw && search.nsfw !== 'both') filter.push(`nsfw = ${search.nsfw}`) if (exists(search.isLive)) filter.push(`isLive = ${search.isLive}`) - if (search.host) filter.push(`account.host = '${search.host}'`) + if (search.host) filter.push(`host = '${search.host}'`) if (search.uuids) addUUIDFilters(filter, search.uuids) if (search.tagsOneOf && search.tagsOneOf.length !== 0) { @@ -63,6 +63,10 @@ export async function queryVideos (search: VideosSearchQuery) { filter.push(clause) } + if (search.hosts && search.hosts.length !== 0) { + filter.push(`host IN ${buildInValuesArray(search.hosts)}`) + } + logger.debug({ filter }, 'Will query Meilisearch for videos.') if (search.boostLanguages && search.boostLanguages.length !== 0) { diff --git a/server/lib/requests/instances-index.ts b/server/lib/requests/instances-index.ts index 6fc58bf..0113747 100644 --- a/server/lib/requests/instances-index.ts +++ b/server/lib/requests/instances-index.ts @@ -1,5 +1,7 @@ import { CONFIG } from '../../initializers/constants' import { doRequest } from '../../helpers/requests' +import memoizee from 'memoizee' +import { logger } from '../../helpers/logger' export async function listIndexInstancesHost (): Promise { const uri = CONFIG.INSTANCES_INDEX.URL @@ -29,3 +31,21 @@ export async function getMajorInstanceVersion (host: string): Promise { return 0 } } + +export const listFilteredInstanceHostsMemoized = memoizee(async (filterTag: string) => { + const uri = CONFIG.INSTANCES_INDEX.URL + + const qs = { + healthy: true, + count: 5000, + filterTag + } + + const { body } = await doRequest({ uri, qs, json: true }) + + const hosts = body.data.map(o => o.host as string) + + logger.info({ hosts }, 'Getting filtered instance hosts for tag ' + filterTag) + + return hosts +}, { maxAge: 60000 }) // 1 minute diff --git a/server/middlewares/validators/search.ts b/server/middlewares/validators/search.ts index 665f9ba..9ca4d0f 100644 --- a/server/middlewares/validators/search.ts +++ b/server/middlewares/validators/search.ts @@ -14,6 +14,8 @@ const commonFiltersValidators = [ .optional() .customSanitizer(toArray) .custom(isStringArray).withMessage('Should have a valid hosts array'), + check('filterTag') + .optional(), (req: express.Request, res: express.Response, next: express.NextFunction) => { logger.debug({ query: req.query, body: req.body }, 'Checking commons filters query') diff --git a/server/types/search-query/common-search.model.ts b/server/types/search-query/common-search.model.ts index f4b71ea..7e449d1 100644 --- a/server/types/search-query/common-search.model.ts +++ b/server/types/search-query/common-search.model.ts @@ -2,4 +2,6 @@ export type CommonSearch = { blockedAccounts?: string[] blockedHosts?: string[] fromHost?: string + hosts?: string[] + filterTag?: string } diff --git a/yarn.lock b/yarn.lock index b300ebf..ada49d6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1311,6 +1311,11 @@ resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.191.tgz#09511e7f7cba275acd8b419ddac8da9a6a79e2fa" integrity sha512-BdZ5BCCvho3EIXw6wUCXHe7rS53AIDPLE+JzwgT+OsJk53oBfbSmZZ7CX4VaRoN78N+TJpFi9QPlfIVNmJYWxQ== +"@types/memoizee@^0.4.11": + version "0.4.11" + resolved "https://registry.yarnpkg.com/@types/memoizee/-/memoizee-0.4.11.tgz#da8897f0064bff3e845b5185e2a323bbd5c8d0a3" + integrity sha512-2gyorIBZu8GoDr9pYjROkxWWcFtHCquF7TVbN2I+/OvgZhnIGQS0vX5KJz4lXNKb8XOSfxFOSG5OLru1ESqLUg== + "@types/memoizee@^0.4.2": version "0.4.8" resolved "https://registry.yarnpkg.com/@types/memoizee/-/memoizee-0.4.8.tgz#04adc0c266a0f5d72db0556fdda2ba17dad9b519" @@ -2026,6 +2031,14 @@ d@1, d@^1.0.1: es5-ext "^0.10.50" type "^1.0.1" +d@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/d/-/d-1.0.2.tgz#2aefd554b81981e7dccf72d6842ae725cb17e5de" + integrity sha512-MOqHvMWF9/9MX6nza0KgvFH4HpMU0EF5uUDXqX/BtxtU8NfB0QzRtJ8Oe/6SuS4kbhyzVJwjd97EA4PKrzJ8bw== + dependencies: + es5-ext "^0.10.64" + type "^2.7.2" + dashdash@^1.12.0: version "1.14.1" resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0" @@ -2242,6 +2255,16 @@ es5-ext@^0.10.35, es5-ext@^0.10.46, es5-ext@^0.10.50, es5-ext@^0.10.53, es5-ext@ es6-symbol "^3.1.3" next-tick "^1.1.0" +es5-ext@^0.10.62, es5-ext@^0.10.64: + version "0.10.64" + resolved "https://registry.yarnpkg.com/es5-ext/-/es5-ext-0.10.64.tgz#12e4ffb48f1ba2ea777f1fcdd1918ef73ea21714" + integrity sha512-p2snDhiLaXe6dahss1LddxqEm+SkuDvV8dnIQG0MWjyHpcMNfXKPE+/Cc0y+PhxJX3A4xGNeFCj5oc0BUh6deg== + dependencies: + es6-iterator "^2.0.3" + es6-symbol "^3.1.3" + esniff "^2.0.1" + next-tick "^1.1.0" + es6-iterator@^2.0.3: version "2.0.3" resolved "https://registry.yarnpkg.com/es6-iterator/-/es6-iterator-2.0.3.tgz#a7de889141a05a94b0854403b2d0a0fbfa98f3b7" @@ -2468,6 +2491,16 @@ eslint@^8.16.0: strip-json-comments "^3.1.0" text-table "^0.2.0" +esniff@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/esniff/-/esniff-2.0.1.tgz#a4d4b43a5c71c7ec51c51098c1d8a29081f9b308" + integrity sha512-kTUIGKQ/mDPFoJ0oVfcmyJn4iBDRptjNVIzwIFR7tqWXdVI9xfA2RMwY/gbSpJG3lkdWNEjLap/NqVHZiJsdfg== + dependencies: + d "^1.0.1" + es5-ext "^0.10.62" + event-emitter "^0.3.5" + type "^2.7.2" + espree@^9.4.0: version "9.4.1" resolved "https://registry.yarnpkg.com/espree/-/espree-9.4.1.tgz#51d6092615567a2c2cff7833445e37c28c0065bd" @@ -3487,6 +3520,20 @@ memoizee@^0.4.14: next-tick "^1.1.0" timers-ext "^0.1.7" +memoizee@^0.4.17: + version "0.4.17" + resolved "https://registry.yarnpkg.com/memoizee/-/memoizee-0.4.17.tgz#942a5f8acee281fa6fb9c620bddc57e3b7382949" + integrity sha512-DGqD7Hjpi/1or4F/aYAspXKNm5Yili0QDAFAY4QYvpqpgiY6+1jOfqpmByzjxbWd/T9mChbCArXAbDAsTm5oXA== + dependencies: + d "^1.0.2" + es5-ext "^0.10.64" + es6-weak-map "^2.0.3" + event-emitter "^0.3.5" + is-promise "^2.2.2" + lru-queue "^0.1.0" + next-tick "^1.1.0" + timers-ext "^0.1.7" + merge-descriptors@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/merge-descriptors/-/merge-descriptors-1.0.1.tgz#b00aaa556dd8b44568150ec9d1b953f3f90cbb61"