Support filter tags from instances.jpt
This commit is contained in:
Chocobozzz
parent
ce10a61b0c
commit
8dfd71904b
|
|
@ -3,7 +3,6 @@
|
|||
## Dev
|
||||
|
||||
```terminal
|
||||
$ git submodule update --init --recursive
|
||||
$ yarn install --pure-lockfile
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@
|
|||
"js-yaml": "^4.1.0",
|
||||
"lodash": "^4.17.15",
|
||||
"meilisearch": "^0.35.0",
|
||||
"memoizee": "^0.4.17",
|
||||
"mkdirp": "^1.0.4",
|
||||
"morgan": "^1.9.1",
|
||||
"multer": "^1.4.5-lts.1",
|
||||
|
|
@ -53,6 +54,7 @@
|
|||
"@types/fluent-ffmpeg": "^2.1.20",
|
||||
"@types/fs-extra": "^9.0.13",
|
||||
"@types/lodash": "^4.14.182",
|
||||
"@types/memoizee": "^0.4.11",
|
||||
"@types/mkdirp": "^1.0.2",
|
||||
"@types/morgan": "^1.9.3",
|
||||
"@types/multer": "^1.4.7",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import { ResultList } from '@peertube/peertube-types'
|
||||
import { CONFIG } from '../../initializers/constants'
|
||||
import { CommonSearch } from '../../types/search-query/common-search.model'
|
||||
import { listFilteredInstanceHostsMemoized } from '../requests/instances-index'
|
||||
|
||||
export class Searcher <T extends CommonSearch, R, F> {
|
||||
|
||||
|
|
@ -20,6 +21,19 @@ export class Searcher <T extends CommonSearch, R, F> {
|
|||
query.blockedHosts = query.blockedHosts.concat(CONFIG.API.BLACKLIST.HOSTS)
|
||||
}
|
||||
|
||||
if (queryArg.filterTag) {
|
||||
if (!Array.isArray(query.hosts)) query.hosts = []
|
||||
|
||||
query.hosts = query.hosts.concat(await listFilteredInstanceHostsMemoized(query.filterTag))
|
||||
|
||||
if (query.hosts.length === 0) {
|
||||
return {
|
||||
total: 0,
|
||||
data: []
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const resultList = await this.queryFn(query)
|
||||
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -25,6 +25,10 @@ export async function queryChannels (search: ChannelsSearchQuery) {
|
|||
filter.push(`handle IN ${buildInValuesArray(search.handles)}`)
|
||||
}
|
||||
|
||||
if (search.hosts && search.hosts.length !== 0) {
|
||||
filter.push(`host IN ${buildInValuesArray(search.hosts)}`)
|
||||
}
|
||||
|
||||
if (search.blockedHosts && search.blockedHosts.length !== 0) {
|
||||
filter.push(`host NOT IN ${buildInValuesArray(search.blockedHosts)}`)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ export async function queryPlaylists (search: PlaylistsSearchQuery) {
|
|||
'videosLength != 0'
|
||||
]
|
||||
|
||||
if (search.host) filter.push(`ownerAccount.host = '${search.host}'`)
|
||||
if (search.host) filter.push(`host = '${search.host}'`)
|
||||
|
||||
if (search.blockedAccounts && search.blockedAccounts.length !== 0) {
|
||||
filter.push(`ownerAccount.handle NOT IN ${buildInValuesArray(search.blockedAccounts)}`)
|
||||
|
|
@ -23,6 +23,10 @@ export async function queryPlaylists (search: PlaylistsSearchQuery) {
|
|||
filter.push(`host NOT IN ${buildInValuesArray(search.blockedHosts)}`)
|
||||
}
|
||||
|
||||
if (search.hosts && search.hosts.length !== 0) {
|
||||
filter.push(`host IN ${buildInValuesArray(search.hosts)}`)
|
||||
}
|
||||
|
||||
if (search.uuids) addUUIDFilters(filter, search.uuids)
|
||||
|
||||
logger.debug({ filter }, 'Will query Meilisearch for playlists.')
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@ export async function queryVideos (search: VideosSearchQuery) {
|
|||
if (search.nsfw && search.nsfw !== 'both') filter.push(`nsfw = ${search.nsfw}`)
|
||||
|
||||
if (exists(search.isLive)) filter.push(`isLive = ${search.isLive}`)
|
||||
if (search.host) filter.push(`account.host = '${search.host}'`)
|
||||
if (search.host) filter.push(`host = '${search.host}'`)
|
||||
if (search.uuids) addUUIDFilters(filter, search.uuids)
|
||||
|
||||
if (search.tagsOneOf && search.tagsOneOf.length !== 0) {
|
||||
|
|
@ -63,6 +63,10 @@ export async function queryVideos (search: VideosSearchQuery) {
|
|||
filter.push(clause)
|
||||
}
|
||||
|
||||
if (search.hosts && search.hosts.length !== 0) {
|
||||
filter.push(`host IN ${buildInValuesArray(search.hosts)}`)
|
||||
}
|
||||
|
||||
logger.debug({ filter }, 'Will query Meilisearch for videos.')
|
||||
|
||||
if (search.boostLanguages && search.boostLanguages.length !== 0) {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
import { CONFIG } from '../../initializers/constants'
|
||||
import { doRequest } from '../../helpers/requests'
|
||||
import memoizee from 'memoizee'
|
||||
import { logger } from '../../helpers/logger'
|
||||
|
||||
export async function listIndexInstancesHost (): Promise<string[]> {
|
||||
const uri = CONFIG.INSTANCES_INDEX.URL
|
||||
|
|
@ -29,3 +31,21 @@ export async function getMajorInstanceVersion (host: string): Promise<number> {
|
|||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
export const listFilteredInstanceHostsMemoized = memoizee(async (filterTag: string) => {
|
||||
const uri = CONFIG.INSTANCES_INDEX.URL
|
||||
|
||||
const qs = {
|
||||
healthy: true,
|
||||
count: 5000,
|
||||
filterTag
|
||||
}
|
||||
|
||||
const { body } = await doRequest<any>({ uri, qs, json: true })
|
||||
|
||||
const hosts = body.data.map(o => o.host as string)
|
||||
|
||||
logger.info({ hosts }, 'Getting filtered instance hosts for tag ' + filterTag)
|
||||
|
||||
return hosts
|
||||
}, { maxAge: 60000 }) // 1 minute
|
||||
|
|
|
|||
|
|
@ -14,6 +14,8 @@ const commonFiltersValidators = [
|
|||
.optional()
|
||||
.customSanitizer(toArray)
|
||||
.custom(isStringArray).withMessage('Should have a valid hosts array'),
|
||||
check('filterTag')
|
||||
.optional(),
|
||||
|
||||
(req: express.Request, res: express.Response, next: express.NextFunction) => {
|
||||
logger.debug({ query: req.query, body: req.body }, 'Checking commons filters query')
|
||||
|
|
|
|||
|
|
@ -2,4 +2,6 @@ export type CommonSearch = {
|
|||
blockedAccounts?: string[]
|
||||
blockedHosts?: string[]
|
||||
fromHost?: string
|
||||
hosts?: string[]
|
||||
filterTag?: string
|
||||
}
|
||||
|
|
|
|||
47
yarn.lock
47
yarn.lock
|
|
@ -1311,6 +1311,11 @@
|
|||
resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.191.tgz#09511e7f7cba275acd8b419ddac8da9a6a79e2fa"
|
||||
integrity sha512-BdZ5BCCvho3EIXw6wUCXHe7rS53AIDPLE+JzwgT+OsJk53oBfbSmZZ7CX4VaRoN78N+TJpFi9QPlfIVNmJYWxQ==
|
||||
|
||||
"@types/memoizee@^0.4.11":
|
||||
version "0.4.11"
|
||||
resolved "https://registry.yarnpkg.com/@types/memoizee/-/memoizee-0.4.11.tgz#da8897f0064bff3e845b5185e2a323bbd5c8d0a3"
|
||||
integrity sha512-2gyorIBZu8GoDr9pYjROkxWWcFtHCquF7TVbN2I+/OvgZhnIGQS0vX5KJz4lXNKb8XOSfxFOSG5OLru1ESqLUg==
|
||||
|
||||
"@types/memoizee@^0.4.2":
|
||||
version "0.4.8"
|
||||
resolved "https://registry.yarnpkg.com/@types/memoizee/-/memoizee-0.4.8.tgz#04adc0c266a0f5d72db0556fdda2ba17dad9b519"
|
||||
|
|
@ -2026,6 +2031,14 @@ d@1, d@^1.0.1:
|
|||
es5-ext "^0.10.50"
|
||||
type "^1.0.1"
|
||||
|
||||
d@^1.0.2:
|
||||
version "1.0.2"
|
||||
resolved "https://registry.yarnpkg.com/d/-/d-1.0.2.tgz#2aefd554b81981e7dccf72d6842ae725cb17e5de"
|
||||
integrity sha512-MOqHvMWF9/9MX6nza0KgvFH4HpMU0EF5uUDXqX/BtxtU8NfB0QzRtJ8Oe/6SuS4kbhyzVJwjd97EA4PKrzJ8bw==
|
||||
dependencies:
|
||||
es5-ext "^0.10.64"
|
||||
type "^2.7.2"
|
||||
|
||||
dashdash@^1.12.0:
|
||||
version "1.14.1"
|
||||
resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0"
|
||||
|
|
@ -2242,6 +2255,16 @@ es5-ext@^0.10.35, es5-ext@^0.10.46, es5-ext@^0.10.50, es5-ext@^0.10.53, es5-ext@
|
|||
es6-symbol "^3.1.3"
|
||||
next-tick "^1.1.0"
|
||||
|
||||
es5-ext@^0.10.62, es5-ext@^0.10.64:
|
||||
version "0.10.64"
|
||||
resolved "https://registry.yarnpkg.com/es5-ext/-/es5-ext-0.10.64.tgz#12e4ffb48f1ba2ea777f1fcdd1918ef73ea21714"
|
||||
integrity sha512-p2snDhiLaXe6dahss1LddxqEm+SkuDvV8dnIQG0MWjyHpcMNfXKPE+/Cc0y+PhxJX3A4xGNeFCj5oc0BUh6deg==
|
||||
dependencies:
|
||||
es6-iterator "^2.0.3"
|
||||
es6-symbol "^3.1.3"
|
||||
esniff "^2.0.1"
|
||||
next-tick "^1.1.0"
|
||||
|
||||
es6-iterator@^2.0.3:
|
||||
version "2.0.3"
|
||||
resolved "https://registry.yarnpkg.com/es6-iterator/-/es6-iterator-2.0.3.tgz#a7de889141a05a94b0854403b2d0a0fbfa98f3b7"
|
||||
|
|
@ -2468,6 +2491,16 @@ eslint@^8.16.0:
|
|||
strip-json-comments "^3.1.0"
|
||||
text-table "^0.2.0"
|
||||
|
||||
esniff@^2.0.1:
|
||||
version "2.0.1"
|
||||
resolved "https://registry.yarnpkg.com/esniff/-/esniff-2.0.1.tgz#a4d4b43a5c71c7ec51c51098c1d8a29081f9b308"
|
||||
integrity sha512-kTUIGKQ/mDPFoJ0oVfcmyJn4iBDRptjNVIzwIFR7tqWXdVI9xfA2RMwY/gbSpJG3lkdWNEjLap/NqVHZiJsdfg==
|
||||
dependencies:
|
||||
d "^1.0.1"
|
||||
es5-ext "^0.10.62"
|
||||
event-emitter "^0.3.5"
|
||||
type "^2.7.2"
|
||||
|
||||
espree@^9.4.0:
|
||||
version "9.4.1"
|
||||
resolved "https://registry.yarnpkg.com/espree/-/espree-9.4.1.tgz#51d6092615567a2c2cff7833445e37c28c0065bd"
|
||||
|
|
@ -3487,6 +3520,20 @@ memoizee@^0.4.14:
|
|||
next-tick "^1.1.0"
|
||||
timers-ext "^0.1.7"
|
||||
|
||||
memoizee@^0.4.17:
|
||||
version "0.4.17"
|
||||
resolved "https://registry.yarnpkg.com/memoizee/-/memoizee-0.4.17.tgz#942a5f8acee281fa6fb9c620bddc57e3b7382949"
|
||||
integrity sha512-DGqD7Hjpi/1or4F/aYAspXKNm5Yili0QDAFAY4QYvpqpgiY6+1jOfqpmByzjxbWd/T9mChbCArXAbDAsTm5oXA==
|
||||
dependencies:
|
||||
d "^1.0.2"
|
||||
es5-ext "^0.10.64"
|
||||
es6-weak-map "^2.0.3"
|
||||
event-emitter "^0.3.5"
|
||||
is-promise "^2.2.2"
|
||||
lru-queue "^0.1.0"
|
||||
next-tick "^1.1.0"
|
||||
timers-ext "^0.1.7"
|
||||
|
||||
merge-descriptors@1.0.1:
|
||||
version "1.0.1"
|
||||
resolved "https://registry.yarnpkg.com/merge-descriptors/-/merge-descriptors-1.0.1.tgz#b00aaa556dd8b44568150ec9d1b953f3f90cbb61"
|
||||
|
|
|
|||
Loading…
Reference in New Issue