Peertube
/
sepia-search-motore-di-ricerca-video
mirror of https://framagit.org/framasoft/peertube/search-index/
1
0
Fork 0
Code Issues Releases Wiki Activity

Move from elastic search to meilisearch

This commit is contained in:
Chocobozzz 2023-11-09 14:34:36 +01:00
parent 6866717bcd
commit 7fb1b791ae
No known key found for this signature in database
GPG Key ID: 583A612D890159BE
43 changed files with 908 additions and 1958 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
README.md
View File

@ -7,7 +7,7 @@ $ git submodule update --init --recursive
$ yarn install --pure-lockfile $ yarn install --pure-lockfile
``` ```
The database (Elastic Search) is automatically created by PeerTube at startup. Indexes in Meilisearch are automatically created by PeerTube at startup.
Run simultaneously (for example with 3 terminals): Run simultaneously (for example with 3 terminals):
@ -33,7 +33,7 @@ Add the locale in `client/src/main.ts` and `client/gettext.config.js` and run `n
Install dependencies: Install dependencies:
* NodeJS (v16) * NodeJS (v16)
* Elastic Search * MeiliSearch
```terminal ```terminal
$ git clone https://framagit.org/framasoft/peertube/search-index.git /var/www/peertube-search-index $ git clone https://framagit.org/framasoft/peertube/search-index.git /var/www/peertube-search-index
@ -44,19 +44,3 @@ $ cp config/default.yaml config/production.yaml
$ vim config/production.yaml $ vim config/production.yaml
$ NODE_ENV=production NODE_CONFIG_DIR=/var/www/peertube-search-index/config node dist/server.js $ NODE_ENV=production NODE_CONFIG_DIR=/var/www/peertube-search-index/config node dist/server.js
``` ```
### Mapping migration
To update Elastic Search index mappings without downtime, run another instance of the search indexer
using the same configuration that the main node. You just have to update `elastic-search.indexes.*` to use new index names.
```
$ cd /var/www/peertube-search-index
$ cp config/production.yaml config/production-1.yaml
$ vim config/production-1.yaml
$ NODE_ENV=production NODE_APP_INSTANCE=1 NODE_CONFIG_DIR=/var/www/peertube-search-index/config node dist/server.js
```
After a while the new indexes will be filled. You can then stop the second indexer, update `config/production.yaml` to use
the new index names and restart the main index.

2
client/src/components/ChannelResult.vue
View File

@ -105,7 +105,7 @@
return this.channel.avatar.url return this.channel.avatar.url
} }
if (this.channel.avatar.length === 0) return '' if (this.channel.avatars.length === 0) return ''
const biggestAvatar = [ ...this.channel.avatars ].sort((a1, a2) => { const biggestAvatar = [ ...this.channel.avatars ].sort((a1, a2) => {
if (a1.width < a2.width) return 1 if (a1.width < a2.width) return 1

98
config/default.yaml
View File

@ -7,17 +7,11 @@ webserver:
hostname: 'localhost' hostname: 'localhost'
port: 3234 port: 3234
elastic-search: meilisearch:
# https or http host: 'http://127.0.0.1:7700'
http: 'http'
auth: api_key: null
username: null
password: null
ssl:
# Specificy a custom CA
ca: null
hostname: 'localhost'
port: 9200
indexes: indexes:
videos: 'peertube-index-videos' videos: 'peertube-index-videos'
channels: 'peertube-index-channels' channels: 'peertube-index-channels'
@ -54,88 +48,6 @@ instances-index:
enabled: false enabled: false
hosts: null hosts: null
videos-search:
# Allow client to send browser language to boost results score that are in these languages
boost-languages:
enabled: true
# Add ability to change videos search fields boost and match value
# See https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-multi-match-query.html for more information
#
# If boost == 0, the field will not be part of the search
#
# match_type could be 'default' or 'phrase'
# * default: use default Elastic Search match query, including fuzziness
# * phrase: use Elastic Search phrase match query
search-fields:
uuid:
boost: 100
match_type: 'default'
short-uuid:
boost: 100
match_type: 'default'
name:
boost: 5
match_type: 'default'
description:
boost: 1
match_type: 'phrase'
tags:
boost: 2
match_type: 'default'
account-display-name:
boost: 2
match_type: 'default'
channel-display-name:
boost: 2
match_type: 'default'
channels-search:
# Add ability to change channels search fields boost and match value
# See https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-multi-match-query.html for more information
#
# If boost == 0, the field will not be part of the search
#
# match_type could be 'default' or 'phrase'
# * default: use default Elastic Search match query, including fuzziness
# * phrase: use Elastic Search phrase match query
search-fields:
name:
boost: 5
match_type: 'default'
description:
boost: 1
match_type: 'phrase'
display-name:
boost: 3
match_type: 'default'
account-display-name:
boost: 2
match_type: 'default'
playlists-search:
# Add ability to change playlists search fields boost and match value
# See https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-multi-match-query.html for more information
#
# If boost == 0, the field will not be part of the search
#
# match_type could be 'default' or 'phrase'
# * default: use default Elastic Search match query, including fuzziness
# * phrase: use Elastic Search phrase match query
search-fields:
uuid:
boost: 100
match_type: 'default'
short-uuid:
boost: 100
match_type: 'default'
display-name:
boost: 5
match_type: 'default'
description:
boost: 1
match_type: 'phrase'
api: api:
# Blacklist hosts that will not be returned by the search API # Blacklist hosts that will not be returned by the search API
blacklist: blacklist:

7
config/test.yaml
View File

@ -1,4 +1,7 @@
elastic-search: meilisearch:
# You can put this key as master key when starting meilisearch
api_key: '4kMeZtP0QsgE3QCDSEMYUt_WFusGjq5JgOc9atujpKw'
indexes: indexes:
videos: 'peertube-index-videos-test1' videos: 'peertube-index-videos-test1'
channels: 'peertube-index-channels-test1' channels: 'peertube-index-channels-test1'
@ -25,9 +28,7 @@ instances-index:
- 'thinkerview.video' - 'thinkerview.video'
- 'replay.jres.org' - 'replay.jres.org'
- 'tube.nah.re' - 'tube.nah.re'
- 'peertube.parleur.net'
- 'video.passageenseine.fr' - 'video.passageenseine.fr'
- 'exode.me'
api: api:
blacklist: blacklist:

3
package.json
View File

@ -22,7 +22,6 @@
"i18n:update": "cd client && git fetch weblate && git merge weblate/master && npm run gettext:extract && npm run gettext:compile" "i18n:update": "cd client && git fetch weblate && git merge weblate/master && npm run gettext:extract && npm run gettext:compile"
}, },
"dependencies": { "dependencies": {
"@elastic/elasticsearch": "^8.2.1",
"async": "^3.2.3", "async": "^3.2.3",
"bluebird": "^3.5.3", "bluebird": "^3.5.3",
"body-parser": "^1.20.0", "body-parser": "^1.20.0",
@ -33,6 +32,7 @@
"fs-extra": "^11.1.0", "fs-extra": "^11.1.0",
"js-yaml": "^4.1.0", "js-yaml": "^4.1.0",
"lodash": "^4.17.15", "lodash": "^4.17.15",
"meilisearch": "^0.35.0",
"mkdirp": "^1.0.4", "mkdirp": "^1.0.4",
"morgan": "^1.9.1", "morgan": "^1.9.1",
"multer": "^1.4.5-lts.1", "multer": "^1.4.5-lts.1",
@ -59,7 +59,6 @@
"@types/node": "^18.11.15", "@types/node": "^18.11.15",
"@types/pino": "^7.0.5", "@types/pino": "^7.0.5",
"@types/request": "^2.48.8", "@types/request": "^2.48.8",
"@types/sequelize": "^4.28.13",
"@types/validator": "^13.7.2", "@types/validator": "^13.7.2",
"@typescript-eslint/eslint-plugin": "^5.27.0", "@typescript-eslint/eslint-plugin": "^5.27.0",
"@typescript-eslint/parser": "^5.27.0", "@typescript-eslint/parser": "^5.27.0",

2
server/controllers/api/search-channels.ts
View File

@ -1,6 +1,6 @@
import express from 'express' import express from 'express'
import { Searcher } from '../../lib/controllers/searcher' import { Searcher } from '../../lib/controllers/searcher'
import { formatChannelForAPI, queryChannels } from '../../lib/elastic-search/elastic-search-channels' import { formatChannelForAPI, queryChannels } from '../../lib/meilisearch/meilisearch-channels'
import { asyncMiddleware } from '../../middlewares/async' import { asyncMiddleware } from '../../middlewares/async'
import { setDefaultPagination } from '../../middlewares/pagination' import { setDefaultPagination } from '../../middlewares/pagination'
import { setDefaultSearchSort } from '../../middlewares/sort' import { setDefaultSearchSort } from '../../middlewares/sort'

2
server/controllers/api/search-playlists.ts
View File

@ -1,6 +1,6 @@
import express from 'express' import express from 'express'
import { Searcher } from '../../lib/controllers/searcher' import { Searcher } from '../../lib/controllers/searcher'
import { formatPlaylistForAPI, queryPlaylists } from '../../lib/elastic-search/elastic-search-playlists' import { formatPlaylistForAPI, queryPlaylists } from '../../lib/meilisearch/meilisearch-playlists'
import { asyncMiddleware } from '../../middlewares/async' import { asyncMiddleware } from '../../middlewares/async'
import { setDefaultPagination } from '../../middlewares/pagination' import { setDefaultPagination } from '../../middlewares/pagination'
import { setDefaultSearchSort } from '../../middlewares/sort' import { setDefaultSearchSort } from '../../middlewares/sort'

2
server/controllers/api/search-videos.ts
View File

@ -1,6 +1,6 @@
import express from 'express' import express from 'express'
import { Searcher } from '../../lib/controllers/searcher' import { Searcher } from '../../lib/controllers/searcher'
import { formatVideoForAPI, queryVideos } from '../../lib/elastic-search/elastic-search-videos' import { formatVideoForAPI, queryVideos } from '../../lib/meilisearch/meilisearch-videos'
import { asyncMiddleware } from '../../middlewares/async' import { asyncMiddleware } from '../../middlewares/async'
import { setDefaultPagination } from '../../middlewares/pagination' import { setDefaultPagination } from '../../middlewares/pagination'
import { setDefaultSearchSort } from '../../middlewares/sort' import { setDefaultSearchSort } from '../../middlewares/sort'

30
server/helpers/elastic-search.ts
View File

@ -1,30 +0,0 @@
import { readFileSync } from 'fs-extra'
import { Client } from '@elastic/elasticsearch'
import { CONFIG } from '../initializers/constants'
const elasticOptions = {
node: CONFIG.ELASTIC_SEARCH.HTTP + '://' + CONFIG.ELASTIC_SEARCH.HOSTNAME + ':' + CONFIG.ELASTIC_SEARCH.PORT
}
if (CONFIG.ELASTIC_SEARCH.SSL.CA) {
Object.assign(elasticOptions, {
tls: {
ca: readFileSync(CONFIG.ELASTIC_SEARCH.SSL.CA)
}
})
}
if (CONFIG.ELASTIC_SEARCH.AUTH.USERNAME) {
Object.assign(elasticOptions, {
auth: {
username: CONFIG.ELASTIC_SEARCH.AUTH.USERNAME,
password: CONFIG.ELASTIC_SEARCH.AUTH.PASSWORD
}
})
}
const elasticSearch = new Client(elasticOptions)
export {
elasticSearch
}

9
server/helpers/meilisearch.ts Normal file
View File

@ -0,0 +1,9 @@
import { MeiliSearch } from 'meilisearch'
import { CONFIG } from '../initializers/constants'
const client = new MeiliSearch({
host: CONFIG.MEILISEARCH.HOST,
apiKey: CONFIG.MEILISEARCH.API_KEY
})
export { client }

154
server/initializers/constants.ts
View File

@ -12,21 +12,13 @@ const CONFIG = {
HOSTNAME: config.get<string>('webserver.hostname'), HOSTNAME: config.get<string>('webserver.hostname'),
PORT: config.get<number>('webserver.port') PORT: config.get<number>('webserver.port')
}, },
ELASTIC_SEARCH: { MEILISEARCH: {
HTTP: config.get<string>('elastic-search.http'), HOST: config.get<string>('meilisearch.host'),
AUTH: { API_KEY: config.get<string>('meilisearch.api_key'),
USERNAME: config.get<string>('elastic-search.auth.username'),
PASSWORD: config.get<string>('elastic-search.auth.password')
},
SSL: {
CA: config.get<string>('elastic-search.ssl.ca')
},
HOSTNAME: config.get<string>('elastic-search.hostname'),
PORT: config.get<number>('elastic-search.port'),
INDEXES: { INDEXES: {
VIDEOS: config.get<string>('elastic-search.indexes.videos'), VIDEOS: config.get<string>('meilisearch.indexes.videos'),
CHANNELS: config.get<string>('elastic-search.indexes.channels'), CHANNELS: config.get<string>('meilisearch.indexes.channels'),
PLAYLISTS: config.get<string>('elastic-search.indexes.playlists') PLAYLISTS: config.get<string>('meilisearch.indexes.playlists')
} }
}, },
LOG: { LOG: {
@ -40,96 +32,6 @@ const CONFIG = {
LEGAL_NOTICES_URL: config.get<string>('search-instance.legal_notices_url'), LEGAL_NOTICES_URL: config.get<string>('search-instance.legal_notices_url'),
THEME: config.get<string>('search-instance.theme') THEME: config.get<string>('search-instance.theme')
}, },
VIDEOS_SEARCH: {
BOOST_LANGUAGES: {
ENABLED: config.get<boolean>('videos-search.boost-languages.enabled')
},
SEARCH_FIELDS: {
UUID: {
FIELD_NAME: 'uuid',
BOOST: config.get<number>('videos-search.search-fields.uuid.boost'),
MATCH_TYPE: config.get<string>('videos-search.search-fields.uuid.match_type')
},
SHORT_UUID: {
FIELD_NAME: 'shortUUID',
BOOST: config.get<number>('videos-search.search-fields.short-uuid.boost'),
MATCH_TYPE: config.get<string>('videos-search.search-fields.short-uuid.match_type')
},
NAME: {
FIELD_NAME: 'name',
BOOST: config.get<number>('videos-search.search-fields.name.boost'),
MATCH_TYPE: config.get<string>('videos-search.search-fields.name.match_type')
},
DESCRIPTION: {
FIELD_NAME: 'description',
BOOST: config.get<number>('videos-search.search-fields.description.boost'),
MATCH_TYPE: config.get<string>('videos-search.search-fields.description.match_type')
},
TAGS: {
FIELD_NAME: 'tags',
BOOST: config.get<number>('videos-search.search-fields.tags.boost'),
MATCH_TYPE: config.get<string>('videos-search.search-fields.tags.match_type')
},
ACCOUNT_DISPLAY_NAME: {
FIELD_NAME: 'account.displayName',
BOOST: config.get<number>('videos-search.search-fields.account-display-name.boost'),
MATCH_TYPE: config.get<string>('videos-search.search-fields.account-display-name.match_type')
},
CHANNEL_DISPLAY_NAME: {
FIELD_NAME: 'channel.displayName',
BOOST: config.get<number>('videos-search.search-fields.channel-display-name.boost'),
MATCH_TYPE: config.get<string>('videos-search.search-fields.channel-display-name.match_type')
}
}
},
CHANNELS_SEARCH: {
SEARCH_FIELDS: {
NAME: {
FIELD_NAME: 'name',
BOOST: config.get<number>('channels-search.search-fields.name.boost'),
MATCH_TYPE: config.get<string>('channels-search.search-fields.name.match_type')
},
DESCRIPTION: {
FIELD_NAME: 'description',
BOOST: config.get<number>('channels-search.search-fields.description.boost'),
MATCH_TYPE: config.get<string>('channels-search.search-fields.description.match_type')
},
DISPLAY_NAME: {
FIELD_NAME: 'displayName',
BOOST: config.get<number>('channels-search.search-fields.display-name.boost'),
MATCH_TYPE: config.get<string>('channels-search.search-fields.display-name.match_type')
},
ACCOUNT_DISPLAY_NAME: {
FIELD_NAME: 'ownerAccount.displayName',
BOOST: config.get<number>('channels-search.search-fields.account-display-name.boost'),
MATCH_TYPE: config.get<string>('channels-search.search-fields.account-display-name.match_type')
}
}
},
PLAYLISTS_SEARCH: {
SEARCH_FIELDS: {
UUID: {
FIELD_NAME: 'uuid',
BOOST: config.get<number>('playlists-search.search-fields.uuid.boost'),
MATCH_TYPE: config.get<string>('playlists-search.search-fields.uuid.match_type')
},
SHORT_UUID: {
FIELD_NAME: 'shortUUID',
BOOST: config.get<number>('playlists-search.search-fields.short-uuid.boost'),
MATCH_TYPE: config.get<string>('playlists-search.search-fields.short-uuid.match_type')
},
DISPLAY_NAME: {
FIELD_NAME: 'displayName',
BOOST: config.get<number>('playlists-search.search-fields.display-name.boost'),
MATCH_TYPE: config.get<string>('playlists-search.search-fields.display-name.match_type')
},
DESCRIPTION: {
FIELD_NAME: 'description',
BOOST: config.get<number>('playlists-search.search-fields.description.boost'),
MATCH_TYPE: config.get<string>('playlists-search.search-fields.description.match_type')
}
}
},
INSTANCES_INDEX: { INSTANCES_INDEX: {
URL: config.get<string>('instances-index.url'), URL: config.get<string>('instances-index.url'),
PUBLIC_URL: config.get<string>('instances-index.public_url'), PUBLIC_URL: config.get<string>('instances-index.public_url'),
@ -147,9 +49,9 @@ const CONFIG = {
} }
const SORTABLE_COLUMNS = { const SORTABLE_COLUMNS = {
VIDEOS_SEARCH: [ 'name', 'duration', 'createdAt', 'publishedAt', 'originallyPublishedAt', 'views', 'likes', 'match' ], VIDEOS_SEARCH: [ '_rankingScore', 'match', 'name', 'duration', 'createdAt', 'publishedAt', 'originallyPublishedAt', 'views', 'likes' ],
CHANNELS_SEARCH: [ 'match', 'displayName', 'createdAt' ], CHANNELS_SEARCH: [ '_rankingScore', 'match', 'displayName', 'createdAt' ],
PLAYLISTS_SEARCH: [ 'match', 'displayName', 'createdAt' ] PLAYLISTS_SEARCH: [ '_rankingScore', 'match', 'displayName', 'createdAt' ]
} }
const PAGINATION_START = { const PAGINATION_START = {
@ -165,7 +67,7 @@ const SCHEDULER_INTERVALS_MS = {
indexation: 60000 * 60 * 24 // 24 hours indexation: 60000 * 60 * 24 // 24 hours
} }
const INDEXER_COUNT = 10 const INDEXER_COUNT = 20
const INDEXER_LIMIT = 500000 const INDEXER_LIMIT = 500000
const INDEXER_HOST_CONCURRENCY = 3 const INDEXER_HOST_CONCURRENCY = 3
@ -176,17 +78,6 @@ const REQUESTS = {
WAIT: 10000 // 10 seconds WAIT: 10000 // 10 seconds
} }
const ELASTIC_SEARCH_QUERY = {
FUZZINESS: 'AUTO:4,7',
OPERATOR: 'OR',
MINIMUM_SHOULD_MATCH: '3<75%',
BOOST_LANGUAGE_VALUE: 1,
MALUS_LANGUAGE_VALUE: 0.5,
VIDEOS_MULTI_MATCH_FIELDS: buildMatchFieldConfig(CONFIG.VIDEOS_SEARCH.SEARCH_FIELDS),
CHANNELS_MULTI_MATCH_FIELDS: buildMatchFieldConfig(CONFIG.CHANNELS_SEARCH.SEARCH_FIELDS),
PLAYLISTS_MULTI_MATCH_FIELDS: buildMatchFieldConfig(CONFIG.PLAYLISTS_SEARCH.SEARCH_FIELDS)
}
function getWebserverUrl () { function getWebserverUrl () {
if (CONFIG.WEBSERVER.PORT === 80 || CONFIG.WEBSERVER.PORT === 443) { if (CONFIG.WEBSERVER.PORT === 80 || CONFIG.WEBSERVER.PORT === 443) {
return CONFIG.WEBSERVER.SCHEME + '://' + CONFIG.WEBSERVER.HOSTNAME return CONFIG.WEBSERVER.SCHEME + '://' + CONFIG.WEBSERVER.HOSTNAME
@ -195,28 +86,6 @@ function getWebserverUrl () {
return CONFIG.WEBSERVER.SCHEME + '://' + CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT return CONFIG.WEBSERVER.SCHEME + '://' + CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT
} }
function buildMatchFieldConfig (fields: { [name: string]: { BOOST: number, FIELD_NAME: string, MATCH_TYPE: string } }) {
const selectFields = (matchType: 'phrase' | 'default') => {
return Object.keys(fields)
.filter(fieldName => fields[fieldName].MATCH_TYPE === matchType)
.map(fieldName => fields[fieldName])
}
const buildMultiMatch = (fields: { BOOST: number, FIELD_NAME: string }[]) => {
return fields.map(fieldObj => {
if (fieldObj.BOOST <= 0) return ''
return `${fieldObj.FIELD_NAME}^${fieldObj.BOOST}`
})
.filter(v => !!v)
}
return {
default: buildMultiMatch(selectFields('default')),
phrase: buildMultiMatch(selectFields('phrase'))
}
}
if (isTestInstance()) { if (isTestInstance()) {
SCHEDULER_INTERVALS_MS.indexation = 1000 * 60 * 5 // 5 minutes SCHEDULER_INTERVALS_MS.indexation = 1000 * 60 * 5 // 5 minutes
} }
@ -234,6 +103,5 @@ export {
INDEXER_HOST_CONCURRENCY, INDEXER_HOST_CONCURRENCY,
INDEXER_COUNT, INDEXER_COUNT,
INDEXER_LIMIT, INDEXER_LIMIT,
REQUESTS, REQUESTS
ELASTIC_SEARCH_QUERY
} }

209
server/lib/elastic-search/elastic-search-channels.ts
View File

@ -1,209 +0,0 @@
import { MappingProperty, PropertyName } from '@elastic/elasticsearch/lib/api/types'
import { elasticSearch } from '../../helpers/elastic-search'
import { logger } from '../../helpers/logger'
import { CONFIG, ELASTIC_SEARCH_QUERY } from '../../initializers/constants'
import { DBChannel, EnhancedVideoChannel, IndexableChannel } from '../../types/channel.model'
import { ChannelsSearchQuery } from '../../types/search-query/channel-search.model'
import { buildSort, extractSearchQueryResult } from './elastic-search-queries'
import { buildChannelOrAccountCommonMapping, buildMultiMatchBool } from './shared'
import {
formatActorImageForAPI,
formatActorImageForDB,
formatActorImagesForAPI,
formatActorImagesForDB
} from './shared/elastic-search-avatar'
async function queryChannels (search: ChannelsSearchQuery) {
const bool: any = {}
const mustNot: any[] = []
const filter: any[] = []
if (search.search) {
Object.assign(bool, buildMultiMatchBool(search.search, ELASTIC_SEARCH_QUERY.CHANNELS_MULTI_MATCH_FIELDS))
}
if (search.blockedAccounts) {
mustNot.push({
terms: {
'ownerAccount.handle': search.blockedAccounts
}
})
}
if (search.blockedHosts) {
mustNot.push({
terms: {
host: search.blockedHosts
}
})
}
mustNot.push({
term: {
videosCount: 0
}
})
if (search.host) {
filter.push({
term: {
host: search.host
}
})
}
if (search.handles) {
filter.push({
terms: {
handle: search.handles
}
})
}
if (filter.length !== 0) {
Object.assign(bool, { filter })
}
if (mustNot.length !== 0) {
Object.assign(bool, { must_not: mustNot })
}
const body = {
from: search.start,
size: search.count,
sort: buildSort(search.sort),
query: { bool }
}
logger.debug({ body }, 'Will query Elastic Search for channels.')
const res = await elasticSearch.search({
index: CONFIG.ELASTIC_SEARCH.INDEXES.CHANNELS,
body
})
return extractSearchQueryResult(res)
}
function formatChannelForAPI (c: DBChannel, fromHost?: string): EnhancedVideoChannel {
return {
id: c.id,
score: c.score,
url: c.url,
name: c.name,
host: c.host,
followingCount: c.followingCount,
followersCount: c.followersCount,
createdAt: c.createdAt,
updatedAt: c.updatedAt,
avatar: formatActorImageForAPI(c.avatar),
avatars: formatActorImagesForAPI(c.avatars, c.avatar),
banner: formatActorImageForAPI(c.banner),
banners: formatActorImagesForAPI(c.banners, c.banner),
displayName: c.displayName,
description: c.description,
support: c.support,
isLocal: fromHost === c.host,
videosCount: c.videosCount || 0,
ownerAccount: {
id: c.ownerAccount.id,
url: c.ownerAccount.url,
displayName: c.ownerAccount.displayName,
description: c.ownerAccount.description,
name: c.ownerAccount.name,
host: c.ownerAccount.host,
followingCount: c.ownerAccount.followingCount,
followersCount: c.ownerAccount.followersCount,
createdAt: c.ownerAccount.createdAt,
updatedAt: c.ownerAccount.updatedAt,
avatar: formatActorImageForAPI(c.ownerAccount.avatar),
avatars: formatActorImagesForAPI(c.ownerAccount.avatars, c.ownerAccount.avatar)
}
}
}
function formatChannelForDB (c: IndexableChannel): DBChannel {
return {
id: c.id,
name: c.name,
host: c.host,
url: c.url,
avatar: formatActorImageForDB(c.avatar, c.host),
avatars: formatActorImagesForDB(c.avatars, c.host),
banner: formatActorImageForDB(c.banner, c.host),
banners: formatActorImagesForDB(c.banners, c.host),
displayName: c.displayName,
indexedAt: new Date(),
followingCount: c.followingCount,
followersCount: c.followersCount,
createdAt: c.createdAt,
updatedAt: c.updatedAt,
description: c.description,
support: c.support,
videosCount: c.videosCount,
handle: `${c.name}@${c.host}`,
ownerAccount: {
id: c.ownerAccount.id,
url: c.ownerAccount.url,
displayName: c.ownerAccount.displayName,
description: c.ownerAccount.description,
name: c.ownerAccount.name,
host: c.ownerAccount.host,
followingCount: c.ownerAccount.followingCount,
followersCount: c.ownerAccount.followersCount,
createdAt: c.ownerAccount.createdAt,
updatedAt: c.ownerAccount.updatedAt,
handle: `${c.ownerAccount.name}@${c.ownerAccount.host}`,
avatar: formatActorImageForDB(c.ownerAccount.avatar, c.ownerAccount.host),
avatars: formatActorImagesForDB(c.ownerAccount.avatars, c.ownerAccount.host)
}
}
}
function buildChannelsMapping () {
const base = buildChannelOrAccountCommonMapping()
Object.assign(base, {
videosCount: {
type: 'long'
},
support: {
type: 'keyword'
},
ownerAccount: {
properties: buildChannelOrAccountCommonMapping()
}
} as Record<PropertyName, MappingProperty>)
return base
}
export {
buildChannelsMapping,
formatChannelForDB,
formatChannelForAPI,
queryChannels
}

91
server/lib/elastic-search/elastic-search-index.ts
View File

@ -1,91 +0,0 @@
import { flatMap } from 'lodash'
import { MappingProperty, PropertyName } from '@elastic/elasticsearch/lib/api/types'
import { elasticSearch } from '../../helpers/elastic-search'
import { logger } from '../../helpers/logger'
import { IndexableDoc } from '../../types/indexable-doc.model'
function buildIndex (name: string, mapping: Record<PropertyName, MappingProperty>) {
logger.info('Initialize %s Elastic Search index.', name)
return elasticSearch.indices.create({
index: name,
body: {
settings: {
number_of_shards: 1,
number_of_replicas: 1
},
mappings: {
properties: mapping
}
}
}).catch(err => {
if (err.name === 'ResponseError' && err.meta?.body?.error.root_cause[0]?.type === 'resource_already_exists_exception') return
throw err
})
}
async function indexDocuments <T extends IndexableDoc> (options: {
objects: T[]
formatter: (o: T) => any
replace: boolean
index: string
}) {
const { objects, formatter, replace, index } = options
const elIdIndex: { [elId: string]: T } = {}
for (const object of objects) {
elIdIndex[object.elasticSearchId] = object
}
const method = replace ? 'index' : 'update'
const body = flatMap(objects, v => {
const doc = formatter(v)
const options = replace
? doc
: { doc, doc_as_upsert: true }
return [
{
[method]: {
_id: v.elasticSearchId,
_index: index
}
},
options
]
})
const result = await elasticSearch.bulk({
index,
body
})
if (result.errors === true) {
const msg = 'Cannot insert data in elastic search.'
logger.error({ err: result }, msg)
throw new Error(msg)
}
const created: T[] = result.items
.map(i => i[method])
.filter(i => i.result === 'created')
.map(i => elIdIndex[i._id])
return { created }
}
function refreshIndex (indexName: string) {
logger.info('Refreshing %s index.', indexName)
return elasticSearch.indices.refresh({ index: indexName })
}
export {
buildIndex,
indexDocuments,
refreshIndex
}

234
server/lib/elastic-search/elastic-search-playlists.ts
View File

@ -1,234 +0,0 @@
import { MappingProperty, PropertyName } from '@elastic/elasticsearch/lib/api/types'
import { elasticSearch } from '../../helpers/elastic-search'
import { logger } from '../../helpers/logger'
import { buildUrl } from '../../helpers/utils'
import { CONFIG, ELASTIC_SEARCH_QUERY } from '../../initializers/constants'
import { DBPlaylist, EnhancedPlaylist, IndexablePlaylist } from '../../types/playlist.model'
import { PlaylistsSearchQuery } from '../../types/search-query/playlist-search.model'
import { buildSort, extractSearchQueryResult } from './elastic-search-queries'
import { addUUIDFilters, buildMultiMatchBool } from './shared'
import { buildChannelOrAccountSummaryMapping, formatActorForDB, formatActorSummaryForAPI } from './shared/elastic-search-actor'
async function queryPlaylists (search: PlaylistsSearchQuery) {
const bool: any = {}
const mustNot: any[] = []
const filter: any[] = []
if (search.search) {
Object.assign(bool, buildMultiMatchBool(search.search, ELASTIC_SEARCH_QUERY.PLAYLISTS_MULTI_MATCH_FIELDS))
}
if (search.blockedAccounts) {
mustNot.push({
terms: {
'ownerAccount.handle': search.blockedAccounts
}
})
}
if (search.blockedHosts) {
mustNot.push({
terms: {
host: search.blockedHosts
}
})
}
mustNot.push({
term: {
videosLength: 0
}
})
if (search.host) {
filter.push({
term: {
'ownerAccount.host': search.host
}
})
}
if (search.uuids) {
addUUIDFilters(filter, search.uuids)
}
if (filter.length !== 0) {
Object.assign(bool, { filter })
}
if (mustNot.length !== 0) {
Object.assign(bool, { must_not: mustNot })
}
const body = {
from: search.start,
size: search.count,
sort: buildSort(search.sort),
query: { bool }
}
logger.debug({ body }, 'Will query Elastic Search for playlists.')
const res = await elasticSearch.search({
index: CONFIG.ELASTIC_SEARCH.INDEXES.PLAYLISTS,
body
})
return extractSearchQueryResult(res)
}
function formatPlaylistForAPI (p: DBPlaylist, fromHost?: string): EnhancedPlaylist {
return {
id: p.id,
uuid: p.uuid,
shortUUID: p.shortUUID,
score: p.score,
isLocal: fromHost === p.host,
url: p.url,
displayName: p.displayName,
description: p.description,
privacy: {
id: p.privacy.id,
label: p.privacy.label
},
videosLength: p.videosLength,
type: {
id: p.type.id,
label: p.type.label
},
thumbnailPath: p.thumbnailPath,
thumbnailUrl: buildUrl(p.host, p.thumbnailPath),
embedPath: p.embedPath,
embedUrl: buildUrl(p.host, p.embedPath),
createdAt: p.createdAt,
updatedAt: p.updatedAt,
ownerAccount: formatActorSummaryForAPI(p.ownerAccount),
videoChannel: formatActorSummaryForAPI(p.videoChannel)
}
}
function formatPlaylistForDB (p: IndexablePlaylist): DBPlaylist {
return {
id: p.id,
uuid: p.uuid,
shortUUID: p.shortUUID,
indexedAt: new Date(),
createdAt: p.createdAt,
updatedAt: p.updatedAt,
host: p.host,
url: p.url,
displayName: p.displayName,
description: p.description,
thumbnailPath: p.thumbnailPath,
embedPath: p.embedPath,
type: {
id: p.type.id,
label: p.type.label
},
privacy: {
id: p.privacy.id,
label: p.privacy.label
},
videosLength: p.videosLength,
ownerAccount: formatActorForDB(p.ownerAccount),
videoChannel: formatActorForDB(p.videoChannel)
}
}
function buildPlaylistsMapping () {
return {
id: {
type: 'long'
},
uuid: {
type: 'keyword'
},
shortUUID: {
type: 'keyword'
},
createdAt: {
type: 'date',
format: 'date_optional_time'
},
updatedAt: {
type: 'date',
format: 'date_optional_time'
},
indexedAt: {
type: 'date',
format: 'date_optional_time'
},
privacy: {
properties: {
id: {
type: 'keyword'
},
label: {
type: 'text'
}
}
},
displayName: {
type: 'text'
},
description: {
type: 'text'
},
thumbnailPath: {
type: 'keyword'
},
embedPath: {
type: 'keyword'
},
url: {
type: 'keyword'
},
host: {
type: 'keyword'
},
videosLength: {
type: 'long'
},
ownerAccount: {
properties: buildChannelOrAccountSummaryMapping()
},
videoChannel: {
properties: buildChannelOrAccountSummaryMapping()
}
} as Record<PropertyName, MappingProperty>
}
export {
formatPlaylistForAPI,
buildPlaylistsMapping,
formatPlaylistForDB,
queryPlaylists
}

134
server/lib/elastic-search/elastic-search-queries.ts
View File

@ -1,134 +0,0 @@
import { difference } from 'lodash'
import { estypes } from '@elastic/elasticsearch'
import { AggregationsBuckets, AggregationsStringTermsAggregate, AggregationsStringTermsBucket } from '@elastic/elasticsearch/lib/api/types'
import { elasticSearch } from '../../helpers/elastic-search'
import { logger } from '../../helpers/logger'
async function removeNotExistingIdsFromHost (indexName: string, host: string, existingIds: Set<number>) {
const idsFromDB = await getIdsOf(indexName, host)
// Elastic Search limits max terms amount
const idsToRemove = difference(idsFromDB, Array.from(existingIds)).splice(0, 50000)
logger.info({ idsToRemove }, 'Will remove %d entries from %s of host %s.', idsToRemove.length, indexName, host)
return elasticSearch.deleteByQuery({
index: indexName,
body: {
query: {
bool: {
filter: [
{
terms: {
id: idsToRemove
}
},
{
term: {
host
}
}
]
}
}
}
})
}
function removeFromHosts (indexName: string, hosts: string[]) {
if (hosts.length === 0) return
logger.info({ hosts }, 'Will remove entries of index %s from hosts.', indexName)
return elasticSearch.deleteByQuery({
index: indexName,
body: {
query: {
bool: {
filter: {
terms: {
host: hosts
}
}
}
}
}
})
}
async function getIdsOf (indexName: string, host: string) {
const res = await elasticSearch.search<unknown, Record<'ids', AggregationsStringTermsAggregate>>({
index: indexName,
body: {
size: 0,
aggs: {
ids: {
terms: {
size: 500000,
field: 'id'
}
}
},
query: {
bool: {
filter: [
{
term: {
host
}
}
]
}
}
}
})
return extractBucketsFromAggregation<number>(res.aggregations.ids.buckets).map(b => b.key)
}
function extractSearchQueryResult (result: estypes.SearchResponse<any, any>) {
const hits = result.hits
return {
total: (hits.total as estypes.SearchTotalHits).value,
data: hits.hits.map(h => Object.assign(h._source, { score: h._score }))
}
}
function extractBucketsFromAggregation <T extends string | number> (buckets: AggregationsBuckets<AggregationsStringTermsBucket>) {
// FIXME: key returned by elastic search can also be a number
return buckets as unknown as { key: T }[]
}
function buildSort (value: string) {
let sortField: string
let direction: 'asc' | 'desc'
if (value.substring(0, 1) === '-') {
direction = 'desc'
sortField = value.substring(1)
} else {
direction = 'asc'
sortField = value
}
const field = sortField === 'match'
? '_score'
: sortField
return [
{
[field]: { order: direction }
}
]
}
export {
elasticSearch,
removeNotExistingIdsFromHost,
getIdsOf,
extractSearchQueryResult,
removeFromHosts,
buildSort,
extractBucketsFromAggregation
}

528
server/lib/elastic-search/elastic-search-videos.ts
View File

@ -1,528 +0,0 @@
import { MappingProperty, PropertyName } from '@elastic/elasticsearch/lib/api/types'
import { exists } from '../../helpers/custom-validators/misc'
import { elasticSearch } from '../../helpers/elastic-search'
import { logger } from '../../helpers/logger'
import { buildUrl } from '../../helpers/utils'
import { CONFIG, ELASTIC_SEARCH_QUERY } from '../../initializers/constants'
import { VideosSearchQuery } from '../../types/search-query/video-search.model'
import { DBVideo, DBVideoDetails, EnhancedVideo, IndexableVideo, IndexableVideoDetails } from '../../types/video.model'
import { buildSort, extractSearchQueryResult } from './elastic-search-queries'
import { addUUIDFilters, buildMultiMatchBool } from './shared'
import { buildChannelOrAccountSummaryMapping, formatActorForDB, formatActorSummaryForAPI } from './shared/elastic-search-actor'
async function queryVideos (search: VideosSearchQuery) {
const bool: any = {}
const filter: any[] = []
const mustNot: any[] = []
if (search.search) {
Object.assign(bool, buildMultiMatchBool(search.search, ELASTIC_SEARCH_QUERY.VIDEOS_MULTI_MATCH_FIELDS))
}
if (search.blockedAccounts) {
mustNot.push({
terms: {
'account.handle': search.blockedAccounts
}
})
}
if (search.blockedHosts) {
mustNot.push({
terms: {
host: search.blockedHosts
}
})
}
if (search.startDate) {
filter.push({
range: {
publishedAt: {
gte: search.startDate
}
}
})
}
if (search.endDate) {
filter.push({
range: {
publishedAt: {
lte: search.endDate
}
}
})
}
if (search.originallyPublishedStartDate) {
filter.push({
range: {
originallyPublishedAt: {
gte: search.startDate
}
}
})
}
if (search.originallyPublishedEndDate) {
filter.push({
range: {
originallyPublishedAt: {
lte: search.endDate
}
}
})
}
if (search.nsfw && search.nsfw !== 'both') {
filter.push({
term: {
nsfw: (search.nsfw + '') === 'true'
}
})
}
if (search.categoryOneOf) {
filter.push({
terms: {
'category.id': search.categoryOneOf
}
})
}
if (search.licenceOneOf) {
filter.push({
terms: {
'licence.id': search.licenceOneOf
}
})
}
if (search.languageOneOf) {
filter.push({
terms: {
'language.id': search.languageOneOf
}
})
}
if (search.tagsOneOf) {
filter.push({
terms: {
tags: search.tagsOneOf.map(t => t.toLowerCase())
}
})
}
if (search.tagsAllOf) {
for (const t of search.tagsAllOf) {
filter.push({
term: {
'tags.raw': {
value: t,
case_insensitive: true
}
}
})
}
}
if (search.durationMin) {
filter.push({
range: {
duration: {
gte: search.durationMin
}
}
})
}
if (search.durationMax) {
filter.push({
range: {
duration: {
lte: search.durationMax
}
}
})
}
if (exists(search.isLive)) {
filter.push({
term: {
isLive: search.isLive
}
})
}
if (search.host) {
filter.push({
term: {
'account.host': search.host
}
})
}
if (search.uuids) {
addUUIDFilters(filter, search.uuids)
}
Object.assign(bool, { filter })
if (mustNot.length !== 0) {
Object.assign(bool, { must_not: mustNot })
}
const body = {
from: search.start,
size: search.count,
sort: buildSort(search.sort)
}
// Allow to boost results depending on query languages
if (
CONFIG.VIDEOS_SEARCH.BOOST_LANGUAGES.ENABLED &&
Array.isArray(search.boostLanguages) &&
search.boostLanguages.length !== 0
) {
const boostScript = `
if (doc['language.id'].size() == 0) {
return _score;
}
String language = doc['language.id'].value;
for (String docLang: params.boostLanguages) {
if (docLang == language) return _score * params.boost;
}
return _score * params.malus;
`
Object.assign(body, {
query: {
script_score: {
query: { bool },
script: {
source: boostScript,
params: {
boostLanguages: search.boostLanguages,
boost: ELASTIC_SEARCH_QUERY.BOOST_LANGUAGE_VALUE,
malus: ELASTIC_SEARCH_QUERY.MALUS_LANGUAGE_VALUE
}
}
}
}
})
} else {
Object.assign(body, { query: { bool } })
}
logger.debug({ body }, 'Will query Elastic Search for videos.')
const res = await elasticSearch.search({
index: CONFIG.ELASTIC_SEARCH.INDEXES.VIDEOS,
body
})
return extractSearchQueryResult(res)
}
function buildVideosMapping () {
return {
id: {
type: 'long'
},
uuid: {
type: 'keyword'
},
shortUUID: {
type: 'keyword'
},
createdAt: {
type: 'date',
format: 'date_optional_time'
},
updatedAt: {
type: 'date',
format: 'date_optional_time'
},
publishedAt: {
type: 'date',
format: 'date_optional_time'
},
originallyPublishedAt: {
type: 'date',
format: 'date_optional_time'
},
indexedAt: {
type: 'date',
format: 'date_optional_time'
},
category: {
properties: {
id: {
type: 'keyword'
},
label: {
type: 'text'
}
}
},
licence: {
properties: {
id: {
type: 'keyword'
},
label: {
type: 'text'
}
}
},
language: {
properties: {
id: {
type: 'keyword'
},
label: {
type: 'text'
}
}
},
privacy: {
properties: {
id: {
type: 'keyword'
},
label: {
type: 'text'
}
}
},
name: {
type: 'text'
},
description: {
type: 'text'
},
tags: {
type: 'text',
fields: {
raw: {
type: 'keyword'
}
}
},
duration: {
type: 'long'
},
thumbnailPath: {
type: 'keyword'
},
previewPath: {
type: 'keyword'
},
embedPath: {
type: 'keyword'
},
url: {
type: 'keyword'
},
views: {
type: 'long'
},
likes: {
type: 'long'
},
dislikes: {
type: 'long'
},
nsfw: {
type: 'boolean'
},
isLive: {
type: 'boolean'
},
host: {
type: 'keyword'
},
account: {
properties: buildChannelOrAccountSummaryMapping()
},
channel: {
properties: buildChannelOrAccountSummaryMapping()
}
} as Record<PropertyName, MappingProperty>
}
function formatVideoForDB (v: IndexableVideo | IndexableVideoDetails): DBVideo | DBVideoDetails {
const video = {
id: v.id,
uuid: v.uuid,
shortUUID: v.shortUUID,
indexedAt: new Date(),
createdAt: v.createdAt,
updatedAt: v.updatedAt,
publishedAt: v.publishedAt,
originallyPublishedAt: v.originallyPublishedAt,
category: {
id: v.category.id,
label: v.category.label
},
licence: {
id: v.licence.id,
label: v.licence.label
},
language: {
id: v.language.id,
label: v.language.label
},
privacy: {
id: v.privacy.id,
label: v.privacy.label
},
name: v.name,
truncatedDescription: v.truncatedDescription,
description: v.description,
waitTranscoding: v.waitTranscoding,
duration: v.duration,
thumbnailPath: v.thumbnailPath,
previewPath: v.previewPath,
embedPath: v.embedPath,
views: v.views,
viewers: v.viewers,
likes: v.likes,
dislikes: v.dislikes,
isLive: v.isLive || false,
nsfw: v.nsfw,
host: v.host,
url: v.url,
files: v.files,
streamingPlaylists: v.streamingPlaylists,
tags: (v as IndexableVideoDetails).tags ? (v as IndexableVideoDetails).tags : undefined,
account: formatActorForDB(v.account),
channel: formatActorForDB(v.channel)
}
if (isVideoDetails(v)) {
return {
...video,
trackerUrls: v.trackerUrls,
descriptionPath: v.descriptionPath,
support: v.support,
commentsEnabled: v.commentsEnabled,
downloadEnabled: v.downloadEnabled
}
}
return video
}
function formatVideoForAPI (v: DBVideoDetails, fromHost?: string): EnhancedVideo {
return {
id: v.id,
uuid: v.uuid,
shortUUID: v.shortUUID,
score: v.score,
createdAt: new Date(v.createdAt),
updatedAt: new Date(v.updatedAt),
publishedAt: new Date(v.publishedAt),
originallyPublishedAt: v.originallyPublishedAt,
category: {
id: v.category.id,
label: v.category.label
},
licence: {
id: v.licence.id,
label: v.licence.label
},
language: {
id: v.language.id,
label: v.language.label
},
privacy: {
id: v.privacy.id,
label: v.privacy.label
},
name: v.name,
description: v.description,
truncatedDescription: v.truncatedDescription,
duration: v.duration,
tags: v.tags,
thumbnailPath: v.thumbnailPath,
thumbnailUrl: buildUrl(v.host, v.thumbnailPath),
previewPath: v.previewPath,
previewUrl: buildUrl(v.host, v.previewPath),
embedPath: v.embedPath,
embedUrl: buildUrl(v.host, v.embedPath),
url: v.url,
isLocal: fromHost && fromHost === v.host,
views: v.views,
viewers: v.viewers,
likes: v.likes,
dislikes: v.dislikes,
isLive: v.isLive,
nsfw: v.nsfw,
account: formatActorSummaryForAPI(v.account),
channel: formatActorSummaryForAPI(v.channel)
}
}
export {
queryVideos,
formatVideoForDB,
formatVideoForAPI,
buildVideosMapping
}
// ---------------------------------------------------------------------------
function isVideoDetails (video: IndexableVideo | IndexableVideoDetails): video is IndexableVideoDetails {
return (video as IndexableVideoDetails).commentsEnabled !== undefined
}

104
server/lib/elastic-search/shared/elastic-search-actor.ts
View File

@ -1,104 +0,0 @@
import { MappingProperty, PropertyName } from '@elastic/elasticsearch/lib/api/types'
import { AccountSummary, VideoChannelSummary } from '@peertube/peertube-types'
import { AdditionalActorAttributes } from '../../../types/actor.model'
import { formatActorImageForDB } from './'
import { buildActorImageMapping, formatActorImageForAPI, formatActorImagesForAPI, formatActorImagesForDB } from './elastic-search-avatar'
function buildChannelOrAccountSummaryMapping () {
return {
id: {
type: 'long'
},
name: {
type: 'text',
fields: {
raw: {
type: 'keyword'
}
}
},
displayName: {
type: 'text'
},
url: {
type: 'keyword'
},
host: {
type: 'keyword'
},
handle: {
type: 'keyword'
},
avatar: {
properties: buildActorImageMapping()
},
// Introduced in 4.2
avatars: {
properties: buildActorImageMapping()
}
} as Record<PropertyName, MappingProperty>
}
function buildChannelOrAccountCommonMapping () {
return {
...buildChannelOrAccountSummaryMapping(),
followingCount: {
type: 'long'
},
followersCount: {
type: 'long'
},
createdAt: {
type: 'date',
format: 'date_optional_time'
},
updatedAt: {
type: 'date',
format: 'date_optional_time'
},
description: {
type: 'text'
}
} as Record<PropertyName, MappingProperty>
}
function formatActorSummaryForAPI (actor: (AccountSummary | VideoChannelSummary) & AdditionalActorAttributes) {
return {
id: actor.id,
name: actor.name,
displayName: actor.displayName,
url: actor.url,
host: actor.host,
avatar: formatActorImageForAPI(actor.avatar),
avatars: formatActorImagesForAPI(actor.avatars, actor.avatar)
}
}
function formatActorForDB (actor: AccountSummary | VideoChannelSummary) {
return {
id: actor.id,
name: actor.name,
displayName: actor.displayName,
url: actor.url,
host: actor.host,
handle: `${actor.name}@${actor.host}`,
avatar: formatActorImageForDB(actor.avatar, actor.host),
avatars: formatActorImagesForDB(actor.avatars, actor.host)
}
}
export {
buildChannelOrAccountCommonMapping,
buildChannelOrAccountSummaryMapping,
formatActorSummaryForAPI,
formatActorForDB
}

77
server/lib/elastic-search/shared/elastic-search-avatar.ts
View File

@ -1,77 +0,0 @@
import { MappingProperty, PropertyName } from '@elastic/elasticsearch/lib/api/types'
import { ActorImage } from '@peertube/peertube-types'
import { buildUrl } from '../../../helpers/utils'
function formatActorImageForAPI (image?: ActorImage) {
if (!image) return null
return {
url: image.url,
path: image.path,
width: image.width,
createdAt: image.createdAt,
updatedAt: image.updatedAt
}
}
function formatActorImagesForAPI (images?: ActorImage[], image?: ActorImage) {
// Does not exist in PeerTube < 4.2
if (!images) {
if (!image) return []
return [ image ]
}
return images.map(a => formatActorImageForAPI(a))
}
// ---------------------------------------------------------------------------
function formatActorImageForDB (image: ActorImage, host: string) {
if (!image) return null
return {
url: buildUrl(host, image.path),
path: image.path,
width: image.width,
createdAt: image.createdAt,
updatedAt: image.updatedAt
}
}
function formatActorImagesForDB (images: ActorImage[], host: string) {
if (!images) return null
return images.map(image => formatActorImageForDB(image, host))
}
// ---------------------------------------------------------------------------
function buildActorImageMapping () {
return {
path: {
type: 'keyword'
},
width: {
type: 'long'
},
createdAt: {
type: 'date',
format: 'date_optional_time'
},
updatedAt: {
type: 'date',
format: 'date_optional_time'
}
} as Record<PropertyName, MappingProperty>
}
export {
formatActorImageForAPI,
formatActorImagesForAPI,
formatActorImageForDB,
formatActorImagesForDB,
buildActorImageMapping
}

3
server/lib/elastic-search/shared/index.ts
View File

@ -1,3 +0,0 @@
export * from './elastic-search-actor'
export * from './elastic-search-avatar'
export * from './query-helpers'

88
server/lib/elastic-search/shared/query-helpers.ts
View File

@ -1,88 +0,0 @@
import validator from 'validator'
import { ELASTIC_SEARCH_QUERY } from '../../../initializers/constants'
function addUUIDFilters (filters: any[], uuids: string[]) {
if (!filters) return
const result = {
shortUUIDs: [] as string[],
uuids: [] as string[]
}
for (const uuid of uuids) {
if (validator.isUUID(uuid)) result.uuids.push(uuid)
else result.shortUUIDs.push(uuid)
}
filters.push({
bool: {
should: [
{
terms: {
uuid: result.uuids
}
},
{
terms: {
shortUUID: result.shortUUIDs
}
}
]
}
})
}
function buildMultiMatchBool (search: string, fieldsObject: { default: string[], phrase: string[] }) {
return {
must: [
{
bool: {
should: [
{
multi_match: {
query: search,
fields: fieldsObject.default,
fuzziness: ELASTIC_SEARCH_QUERY.FUZZINESS,
operator: ELASTIC_SEARCH_QUERY.OPERATOR,
minimum_should_match: ELASTIC_SEARCH_QUERY.MINIMUM_SHOULD_MATCH
}
},
{
multi_match: {
query: search,
fields: fieldsObject.default,
operator: ELASTIC_SEARCH_QUERY.OPERATOR,
minimum_should_match: ELASTIC_SEARCH_QUERY.MINIMUM_SHOULD_MATCH,
type: 'cross_fields'
}
},
{
multi_match: {
query: search,
fields: fieldsObject.phrase,
type: 'phrase'
}
}
]
}
}
],
should: [
// Better score for exact search
{
multi_match: {
query: search,
fields: [ ...fieldsObject.default, ...fieldsObject.phrase ],
operator: ELASTIC_SEARCH_QUERY.OPERATOR,
minimum_should_match: ELASTIC_SEARCH_QUERY.MINIMUM_SHOULD_MATCH
}
}
]
}
}
export {
addUUIDFilters,
buildMultiMatchBool
}

23
server/lib/indexers/channel-indexer.ts
View File

@ -1,20 +1,19 @@
import { logger } from '../../helpers/logger' import { logger } from '../../helpers/logger'
import { CONFIG } from '../../initializers/constants' import { CONFIG, SORTABLE_COLUMNS } from '../../initializers/constants'
import { DBChannel, IndexableChannel } from '../../types/channel.model' import { DBChannel, IndexableChannel } from '../../types/channel.model'
import { buildChannelsMapping, formatChannelForDB } from '../elastic-search/elastic-search-channels' import { formatChannelForDB } from '../meilisearch/meilisearch-channels'
import { getChannel } from '../requests/peertube-instance' import { getChannel } from '../requests/peertube-instance'
import { AbstractIndexer } from './shared' import { AbstractIndexer } from './shared'
export class ChannelIndexer extends AbstractIndexer <IndexableChannel, DBChannel> { export class ChannelIndexer extends AbstractIndexer <IndexableChannel, DBChannel> {
protected readonly primaryKey = 'primaryKey'
protected readonly filterableAttributes = [ 'url', 'host', 'videosCount', 'ownerAccount.handle', 'handle' ]
protected readonly sortableAttributes = SORTABLE_COLUMNS.CHANNELS_SEARCH
// Keep the order, most important first
protected readonly searchableAttributes = [ 'name', 'displayName', 'ownerAccount.displayName', 'description' ]
constructor () { constructor () {
super(CONFIG.ELASTIC_SEARCH.INDEXES.CHANNELS, formatChannelForDB) super(CONFIG.MEILISEARCH.INDEXES.CHANNELS, formatChannelForDB)
this.indexQueue.drain(async () => {
logger.info('Refresh channels index.')
await this.refreshIndex()
})
} }
async indexSpecificElement (host: string, name: string) { async indexSpecificElement (host: string, name: string) {
@ -22,10 +21,6 @@ export class ChannelIndexer extends AbstractIndexer <IndexableChannel, DBChannel
logger.info('Indexing specific channel %s@%s.', name, host) logger.info('Indexing specific channel %s@%s.', name, host)
return this.indexElements([ channel ], true) return this.indexElements([ channel ])
}
buildMapping () {
return buildChannelsMapping()
} }
} }

15
server/lib/indexers/playlist-indexer.ts
View File

@ -1,20 +1,21 @@
import { CONFIG } from '../../initializers/constants' import { CONFIG, SORTABLE_COLUMNS } from '../../initializers/constants'
import { DBPlaylist, IndexablePlaylist } from '../../types/playlist.model' import { DBPlaylist, IndexablePlaylist } from '../../types/playlist.model'
import { buildPlaylistsMapping, formatPlaylistForDB } from '../elastic-search/elastic-search-playlists' import { formatPlaylistForDB } from '../meilisearch/meilisearch-playlists'
import { AbstractIndexer } from './shared' import { AbstractIndexer } from './shared'
export class PlaylistIndexer extends AbstractIndexer <IndexablePlaylist, DBPlaylist> { export class PlaylistIndexer extends AbstractIndexer <IndexablePlaylist, DBPlaylist> {
protected readonly primaryKey = 'uuid'
protected readonly filterableAttributes = [ 'uuid', 'host', 'videosLength' ]
protected readonly sortableAttributes = SORTABLE_COLUMNS.PLAYLISTS_SEARCH
// Keep the order, most important first
protected readonly searchableAttributes = [ 'displayName', 'videoChannel.displayName', 'ownerAccount.displayName', 'description' ]
constructor () { constructor () {
super(CONFIG.ELASTIC_SEARCH.INDEXES.PLAYLISTS, formatPlaylistForDB) super(CONFIG.MEILISEARCH.INDEXES.PLAYLISTS, formatPlaylistForDB)
} }
async indexSpecificElement (host: string, uuid: string) { async indexSpecificElement (host: string, uuid: string) {
// We don't need to index a specific element yet, since we have all playlist information in the list endpoint // We don't need to index a specific element yet, since we have all playlist information in the list endpoint
throw new Error('Not implemented') throw new Error('Not implemented')
} }
buildMapping () {
return buildPlaylistsMapping()
}
} }

62
server/lib/indexers/shared/abstract-indexer.ts
View File

@ -1,11 +1,10 @@
import { AsyncQueue, queue } from 'async' import { AsyncQueue, queue } from 'async'
import { inspect } from 'util' import { inspect } from 'util'
import { MappingProperty, PropertyName } from '@elastic/elasticsearch/lib/api/types'
import { logger } from '../../../helpers/logger' import { logger } from '../../../helpers/logger'
import { INDEXER_QUEUE_CONCURRENCY } from '../../../initializers/constants' import { INDEXER_QUEUE_CONCURRENCY } from '../../../initializers/constants'
import { buildIndex, indexDocuments, refreshIndex } from '../../../lib/elastic-search/elastic-search-index'
import { removeFromHosts, removeNotExistingIdsFromHost } from '../../../lib/elastic-search/elastic-search-queries'
import { IndexableDoc } from '../../../types/indexable-doc.model' import { IndexableDoc } from '../../../types/indexable-doc.model'
import { client } from '../../../helpers/meilisearch'
import { buildInValuesArray } from '../../meilisearch/meilisearch-queries'
// identifier could be an uuid, an handle or a url for example // identifier could be an uuid, an handle or a url for example
export type QueueParam = { host: string, identifier: string } export type QueueParam = { host: string, identifier: string }
@ -13,8 +12,14 @@ export type QueueParam = { host: string, identifier: string }
export abstract class AbstractIndexer <T extends IndexableDoc, DB> { export abstract class AbstractIndexer <T extends IndexableDoc, DB> {
protected readonly indexQueue: AsyncQueue<QueueParam> protected readonly indexQueue: AsyncQueue<QueueParam>
protected abstract readonly primaryKey: keyof DB
protected abstract readonly filterableAttributes: string[]
protected abstract readonly sortableAttributes: string[]
protected abstract readonly searchableAttributes: string[]
protected readonly rankingRules: string[]
abstract indexSpecificElement (host: string, uuid: string): Promise<any> abstract indexSpecificElement (host: string, uuid: string): Promise<any>
abstract buildMapping (): Record<PropertyName, MappingProperty>
constructor ( constructor (
protected readonly indexName: string, protected readonly indexName: string,
@ -33,8 +38,25 @@ export abstract class AbstractIndexer <T extends IndexableDoc, DB> {
}, INDEXER_QUEUE_CONCURRENCY) }, INDEXER_QUEUE_CONCURRENCY)
} }
initIndex () { async initIndex () {
return buildIndex(this.indexName, this.buildMapping()) const { results } = await client.getIndexes()
if (results.some(r => r.uid === this.indexName)) {
logger.info(this.indexName + ' already exists, skipping configuration')
return
}
logger.info('Creating and configuring index ' + this.indexName)
await client.index(this.indexName).updateSearchableAttributes(this.searchableAttributes)
await client.index(this.indexName).updateFilterableAttributes(this.filterableAttributes)
await client.index(this.indexName).updateSortableAttributes(this.sortableAttributes)
await client.index(this.indexName).updateFaceting({ maxValuesPerFacet: 10_000 })
await client.index(this.indexName).updatePagination({ maxTotalHits: 10_000 })
if (this.rankingRules) {
await client.index(this.indexName).updateRankingRules(this.rankingRules)
}
} }
scheduleIndexation (host: string, identifier: string) { scheduleIndexation (host: string, identifier: string) {
@ -42,24 +64,22 @@ export abstract class AbstractIndexer <T extends IndexableDoc, DB> {
.catch(err => logger.error({ err: inspect(err) }, 'Cannot schedule indexation of %s for %s', identifier, host)) .catch(err => logger.error({ err: inspect(err) }, 'Cannot schedule indexation of %s for %s', identifier, host))
} }
refreshIndex () { removeNotExisting (host: string, existingPrimaryKeys: Set<string>) {
return refreshIndex(this.indexName) return client.index(this.indexName).deleteDocuments({
} filter: `${this.primaryKey.toString()} NOT IN ${buildInValuesArray(Array.from(existingPrimaryKeys))} AND host = ${host}`
})
removeNotExisting (host: string, existingIds: Set<number>) {
return removeNotExistingIdsFromHost(this.indexName, host, existingIds)
} }
removeFromHosts (hosts: string[]) { removeFromHosts (hosts: string[]) {
return removeFromHosts(this.indexName, hosts) return client.index(this.indexName).deleteDocuments({
} filter: 'host IN ' + buildInValuesArray(Array.from(hosts))
indexElements (elements: T[], replace = false) {
return indexDocuments({
objects: elements,
formatter: v => this.formatterFn(v),
replace,
index: this.indexName
}) })
} }
async indexElements (elements: T[]) {
const documents = elements.map(e => this.formatterFn(e))
const result = await client.index(this.indexName).updateDocuments(documents, { primaryKey: this.primaryKey.toString() })
logger.debug(result, 'Indexed ' + documents.length + ' documents in ' + this.indexName)
}
} }

56
server/lib/indexers/video-indexer.ts
View File

@ -1,14 +1,54 @@
import { AsyncQueue } from 'async'
import { logger } from '../../helpers/logger' import { logger } from '../../helpers/logger'
import { CONFIG } from '../../initializers/constants' import { AbstractIndexer, QueueParam } from './shared'
import { DBVideo, IndexableVideo } from '../../types/video.model' import { CONFIG, SORTABLE_COLUMNS } from '../../initializers/constants'
import { buildVideosMapping, formatVideoForDB } from '../elastic-search/elastic-search-videos' import { formatVideoForDB } from '../meilisearch/meilisearch-videos'
import { getVideo } from '../requests/peertube-instance' import { getVideo } from '../requests/peertube-instance'
import { AbstractIndexer } from './shared' import { DBVideo, IndexableVideo } from '../../types/video.model'
export class VideoIndexer extends AbstractIndexer <IndexableVideo, DBVideo> { export class VideoIndexer extends AbstractIndexer <IndexableVideo, DBVideo> {
protected readonly indexQueue: AsyncQueue<QueueParam>
protected readonly primaryKey = 'uuid'
protected readonly filterableAttributes = [
'uuid',
'host',
'account.handle',
'account.host',
'publishedAt',
'originallyPublishedAt',
'nsfw',
'category.id',
'licence.id',
'language.id',
'tags',
'duration',
'isLive'
]
protected readonly sortableAttributes = SORTABLE_COLUMNS.VIDEOS_SEARCH
// Keep the order, most important first
protected readonly searchableAttributes = [
'name',
'tags',
'account.displayName',
'channel.displayName',
'description'
]
protected readonly rankingRules = [
"words",
"typo",
"proximity",
"attribute",
"sort",
"exactness",
'language:asc',
'views:desc'
]
constructor () { constructor () {
super(CONFIG.ELASTIC_SEARCH.INDEXES.VIDEOS, formatVideoForDB) super(CONFIG.MEILISEARCH.INDEXES.VIDEOS, formatVideoForDB)
} }
async indexSpecificElement (host: string, uuid: string) { async indexSpecificElement (host: string, uuid: string) {
@ -16,10 +56,6 @@ export class VideoIndexer extends AbstractIndexer <IndexableVideo, DBVideo> {
logger.info('Indexing specific video %s of %s.', uuid, host) logger.info('Indexing specific video %s of %s.', uuid, host)
return this.indexElements([ video ], true) return this.indexElements([ video ])
}
buildMapping () {
return buildVideosMapping()
} }
} }

141
server/lib/meilisearch/meilisearch-channels.ts Normal file
View File

@ -0,0 +1,141 @@
import { createHash } from 'crypto'
import { logger } from '../../helpers/logger'
import { client } from '../../helpers/meilisearch'
import { CONFIG } from '../../initializers/constants'
import { DBChannel, APIVideoChannel, IndexableChannel } from '../../types/channel.model'
import { ChannelsSearchQuery } from '../../types/search-query/channel-search.model'
import { buildInValuesArray, buildSort, extractSearchQueryResult } from './meilisearch-queries'
import {
formatActorImageForAPI,
formatActorImageForDB,
formatActorImagesForAPI,
formatActorImagesForDB
} from './shared/meilisearch-avatar'
export async function queryChannels (search: ChannelsSearchQuery) {
const filter: string[] = [ 'videosCount != 0' ]
if (search.host) filter.push(`host = '${search.host}'`)
if (search.blockedAccounts) filter.push(`ownerAccount.handle NOT IN ${buildInValuesArray(search.blockedAccounts)}`)
if (search.handles) filter.push(`handle IN ${buildInValuesArray(search.handles)}`)
if (search.blockedHosts) filter.push(`host NOT IN ${buildInValuesArray(search.blockedHosts)}`)
logger.debug({ filter }, 'Will query Meilisearch for channels.')
const result = await client.index(CONFIG.MEILISEARCH.INDEXES.CHANNELS).search(search.search, {
offset: search.start,
limit: search.count,
sort: buildSort(search.sort),
showRankingScore: true,
filter
})
return extractSearchQueryResult(result)
}
export function formatChannelForAPI (c: DBChannel, fromHost?: string): APIVideoChannel {
return {
id: c.id,
score: c._rankingScore,
url: c.url,
name: c.name,
host: c.host,
followingCount: c.followingCount,
followersCount: c.followersCount,
createdAt: new Date(c.createdAt),
updatedAt: new Date(c.updatedAt),
avatar: formatActorImageForAPI(c.avatar),
avatars: formatActorImagesForAPI(c.avatars, c.avatar),
banner: formatActorImageForAPI(c.banner),
banners: formatActorImagesForAPI(c.banners, c.banner),
displayName: c.displayName,
description: c.description,
support: c.support,
isLocal: fromHost === c.host,
videosCount: c.videosCount || 0,
ownerAccount: {
id: c.ownerAccount.id,
url: c.ownerAccount.url,
displayName: c.ownerAccount.displayName,
description: c.ownerAccount.description,
name: c.ownerAccount.name,
host: c.ownerAccount.host,
followingCount: c.ownerAccount.followingCount,
followersCount: c.ownerAccount.followersCount,
createdAt: new Date(c.ownerAccount.createdAt),
updatedAt: new Date(c.ownerAccount.updatedAt),
avatar: formatActorImageForAPI(c.ownerAccount.avatar),
avatars: formatActorImagesForAPI(c.ownerAccount.avatars, c.ownerAccount.avatar)
}
}
}
export function formatChannelForDB (c: IndexableChannel): DBChannel {
return {
primaryKey: buildDBChannelPrimaryKey(c),
id: c.id,
name: c.name,
host: c.host,
url: c.url,
avatar: formatActorImageForDB(c.avatar, c.host),
avatars: formatActorImagesForDB(c.avatars, c.host),
banner: formatActorImageForDB(c.banner, c.host),
banners: formatActorImagesForDB(c.banners, c.host),
displayName: c.displayName,
indexedAt: new Date().getTime(),
createdAt: new Date(c.createdAt).getTime(),
updatedAt: new Date(c.updatedAt).getTime(),
followingCount: c.followingCount,
followersCount: c.followersCount,
description: c.description,
support: c.support,
videosCount: c.videosCount,
handle: `${c.name}@${c.host}`,
ownerAccount: {
id: c.ownerAccount.id,
url: c.ownerAccount.url,
displayName: c.ownerAccount.displayName,
description: c.ownerAccount.description,
name: c.ownerAccount.name,
host: c.ownerAccount.host,
followingCount: c.ownerAccount.followingCount,
followersCount: c.ownerAccount.followersCount,
createdAt: new Date(c.ownerAccount.createdAt).getTime(),
updatedAt: new Date(c.ownerAccount.updatedAt).getTime(),
handle: `${c.ownerAccount.name}@${c.ownerAccount.host}`,
avatar: formatActorImageForDB(c.ownerAccount.avatar, c.ownerAccount.host),
avatars: formatActorImagesForDB(c.ownerAccount.avatars, c.ownerAccount.host)
}
}
}
export function buildDBChannelPrimaryKey (c: { id: number, host: string }) {
return `${c.id}_${md5(c.host)}`
}
// Collisions are fine, we just want to generate a primary key in an efficient way
function md5 (value: string) {
return createHash('md5').update(value).digest('hex')
}

27
server/lib/elastic-search/elastic-search-instances.ts → server/lib/meilisearch/meilisearch-instances.ts
View File

@ -1,8 +1,6 @@
import { AggregationsStringTermsAggregate } from '@elastic/elasticsearch/lib/api/types'
import { elasticSearch } from '../../helpers/elastic-search'
import { CONFIG } from '../../initializers/constants' import { CONFIG } from '../../initializers/constants'
import { listIndexInstancesHost } from '../requests/instances-index' import { listIndexInstancesHost } from '../requests/instances-index'
import { extractBucketsFromAggregation } from './elastic-search-queries' import { client } from '../../helpers/meilisearch'
async function buildInstanceHosts () { async function buildInstanceHosts () {
let indexHosts = await listIndexInstancesHost() let indexHosts = await listIndexInstancesHost()
@ -30,28 +28,15 @@ export {
async function listDBInstances () { async function listDBInstances () {
const setResult = new Set<string>() const setResult = new Set<string>()
const indexes = [ const indexes = [
CONFIG.ELASTIC_SEARCH.INDEXES.VIDEOS, CONFIG.MEILISEARCH.INDEXES.VIDEOS,
CONFIG.ELASTIC_SEARCH.INDEXES.CHANNELS CONFIG.MEILISEARCH.INDEXES.CHANNELS
] ]
for (const index of indexes) { for (const index of indexes) {
const res = await elasticSearch.search<unknown, Record<'hosts', AggregationsStringTermsAggregate>>({ const result = await client.index(index).searchForFacetValues({ facetName: 'host' })
index,
body: {
size: 0,
aggs: {
hosts: {
terms: {
size: 5000,
field: 'host'
}
}
}
}
})
for (const b of extractBucketsFromAggregation<string>(res.aggregations.hosts.buckets)) { for (const b of result.facetHits) {
setResult.add(b.key) setResult.add(b.value)
} }
} }

108
server/lib/meilisearch/meilisearch-playlists.ts Normal file
View File

@ -0,0 +1,108 @@
import { logger } from '../../helpers/logger'
import { client } from '../../helpers/meilisearch'
import { buildUrl } from '../../helpers/utils'
import { CONFIG } from '../../initializers/constants'
import { DBPlaylist, APIPlaylist, IndexablePlaylist } from '../../types/playlist.model'
import { PlaylistsSearchQuery } from '../../types/search-query/playlist-search.model'
import { buildInValuesArray, buildSort, extractSearchQueryResult } from './meilisearch-queries'
import { addUUIDFilters } from './shared'
import { formatActorForDB, formatActorSummaryForAPI } from './shared/meilisearch-actor'
export async function queryPlaylists (search: PlaylistsSearchQuery) {
const filter: string[] = [
'videosLength != 0'
]
if (search.host) filter.push(`ownerAccount.host = '${search.host}'`)
if (search.blockedAccounts) filter.push(`ownerAccount.handle NOT IN ${buildInValuesArray(search.blockedAccounts)}`)
if (search.blockedHosts) filter.push(`host NOT IN ${buildInValuesArray(search.blockedHosts)}`)
if (search.uuids) addUUIDFilters(filter, search.uuids)
logger.debug({ filter }, 'Will query Meilisearch for playlists.')
const result = await client.index(CONFIG.MEILISEARCH.INDEXES.PLAYLISTS).search(search.search, {
offset: search.start,
limit: search.count,
sort: buildSort(search.sort),
showRankingScore: true,
filter
})
return extractSearchQueryResult(result)
}
export function formatPlaylistForAPI (p: DBPlaylist, fromHost?: string): APIPlaylist {
return {
id: p.id,
uuid: p.uuid,
shortUUID: p.shortUUID,
score: p._rankingScore,
isLocal: fromHost === p.host,
url: p.url,
displayName: p.displayName,
description: p.description,
privacy: {
id: p.privacy.id,
label: p.privacy.label
},
videosLength: p.videosLength,
type: {
id: p.type.id,
label: p.type.label
},
thumbnailPath: p.thumbnailPath,
thumbnailUrl: buildUrl(p.host, p.thumbnailPath),
embedPath: p.embedPath,
embedUrl: buildUrl(p.host, p.embedPath),
createdAt: new Date(p.createdAt),
updatedAt: new Date(p.updatedAt),
ownerAccount: formatActorSummaryForAPI(p.ownerAccount),
videoChannel: formatActorSummaryForAPI(p.videoChannel)
}
}
export function formatPlaylistForDB (p: IndexablePlaylist): DBPlaylist {
return {
id: p.id,
uuid: p.uuid,
shortUUID: p.shortUUID,
indexedAt: new Date().getTime(),
createdAt: new Date(p.createdAt).getTime(),
updatedAt: new Date(p.updatedAt).getTime(),
host: p.host,
url: p.url,
displayName: p.displayName,
description: p.description,
thumbnailPath: p.thumbnailPath,
embedPath: p.embedPath,
type: {
id: p.type.id,
label: p.type.label
},
privacy: {
id: p.privacy.id,
label: p.privacy.label
},
videosLength: p.videosLength,
ownerAccount: formatActorForDB(p.ownerAccount),
videoChannel: formatActorForDB(p.videoChannel)
}
}

27
server/lib/meilisearch/meilisearch-queries.ts Normal file
View File

@ -0,0 +1,27 @@
import { SearchResponse } from 'meilisearch'
export function extractSearchQueryResult (result: SearchResponse<any, any>) {
return {
total: result.estimatedTotalHits,
data: result.hits
}
}
export function buildSort (value: string) {
if (value === '-match') return [ '_rankingScore:desc' ]
if (value === 'match') return [ '_rankingScore:asc' ]
if (value.substring(0, 1) === '-') {
return [ `${value.substring(1)}:desc`, '_rankingScore:desc' ]
}
return [ `${value}:asc`, '_rankingScore:desc' ]
}
export function buildInQuery (term: string, values: string[] | number[]) {
return `${term} IN ${buildInValuesArray(values)}`
}
export function buildInValuesArray (values: string[] | number[]) {
return '[' + values.map(v => `'${v}'`).join(',') + ']'
}

226
server/lib/meilisearch/meilisearch-videos.ts Normal file
View File

@ -0,0 +1,226 @@
import { exists } from '../../helpers/custom-validators/misc'
import { logger } from '../../helpers/logger'
import { client } from '../../helpers/meilisearch'
import { buildUrl } from '../../helpers/utils'
import { CONFIG } from '../../initializers/constants'
import { VideosSearchQuery } from '../../types/search-query/video-search.model'
import { DBVideo, DBVideoDetails, APIVideo, IndexableVideo, IndexableVideoDetails } from '../../types/video.model'
import { buildInQuery, buildInValuesArray, buildSort, extractSearchQueryResult } from './meilisearch-queries'
import { addUUIDFilters } from './shared'
import { formatActorForDB, formatActorSummaryForAPI } from './shared/meilisearch-actor'
export async function queryVideos (search: VideosSearchQuery) {
const filter: string[] = []
if (search.blockedAccounts) filter.push(`account.handle NOT IN ${buildInValuesArray(search.blockedAccounts)}`)
if (search.blockedHosts) filter.push(`host NOT IN ${buildInValuesArray(search.blockedHosts)}`)
if (search.startDate) filter.push(`publishedAt >= ${new Date(search.startDate).getTime()}`)
if (search.endDate) filter.push(`publishedAt <= ${new Date(search.endDate).getTime()}`)
if (search.originallyPublishedStartDate) {
filter.push(`originallyPublishedAt >= ${new Date(search.originallyPublishedStartDate).getTime()}`)
}
if (search.originallyPublishedEndDate) {
filter.push(`originallyPublishedAt <= ${new Date(search.originallyPublishedEndDate).getTime()}`)
}
if (search.categoryOneOf) filter.push(`category.id IN ${buildInValuesArray(search.categoryOneOf)}`)
if (search.licenceOneOf) filter.push(`licence.id IN ${buildInValuesArray(search.licenceOneOf)}`)
if (search.languageOneOf) filter.push(`language.id IN ${buildInValuesArray(search.languageOneOf)}`)
if (search.durationMin) filter.push(`duration >= ${search.durationMin}`)
if (search.durationMax) filter.push(`duration <= ${search.durationMax}`)
if (search.nsfw && search.nsfw !== 'both') filter.push(`nsfw = ${search.nsfw}`)
if (exists(search.isLive)) filter.push(`isLive = ${search.isLive}`)
if (search.host) filter.push(`account.host = '${search.host}'`)
if (search.uuids) addUUIDFilters(filter, search.uuids)
if (search.tagsOneOf) {
const tagsOneOf = search.tagsOneOf.map(t => t.toLowerCase())
filter.push(`tags IN ${buildInValuesArray(tagsOneOf)}`)
}
if (search.tagsAllOf) {
const tagsAllOf = search.tagsAllOf.map(t => t.toLowerCase())
const clause = tagsAllOf.map(tag => `tags = '${tag}'`).join(' AND ')
filter.push(clause)
}
logger.debug({ filter }, 'Will query Meilisearch for videos.')
const result = await client.index(CONFIG.MEILISEARCH.INDEXES.VIDEOS).search(search.search, {
offset: search.start,
limit: search.count,
sort: buildSort(search.sort),
showRankingScore: true,
filter: [
...filter,
`language.id IN ${buildInValuesArray(search.boostLanguages)} OR language.id IS NULL`
]
})
return extractSearchQueryResult(result)
}
export async function getVideosUpdatedAt (uuids: string[]): Promise<{ updatedAt: number, uuid: string }[]> {
const result = await client.index(CONFIG.MEILISEARCH.INDEXES.VIDEOS).getDocuments({
fields: [ 'updatedAt', 'uuid' ],
filter: [ buildInQuery('uuid', uuids) ]
})
return result.results
}
export function formatVideoForDB (v: IndexableVideo | IndexableVideoDetails): DBVideo | DBVideoDetails {
const video = {
id: v.id,
uuid: v.uuid,
shortUUID: v.shortUUID,
indexedAt: new Date().getTime(),
createdAt: new Date(v.createdAt).getTime(),
updatedAt: new Date(v.updatedAt).getTime(),
publishedAt: new Date(v.publishedAt).getTime(),
originallyPublishedAt: new Date(v.originallyPublishedAt).getTime(),
category: {
id: v.category.id,
label: v.category.label
},
licence: {
id: v.licence.id,
label: v.licence.label
},
language: {
id: v.language.id,
label: v.language.label
},
privacy: {
id: v.privacy.id,
label: v.privacy.label
},
name: v.name,
truncatedDescription: v.truncatedDescription,
description: v.description,
waitTranscoding: v.waitTranscoding,
duration: v.duration,
thumbnailPath: v.thumbnailPath,
previewPath: v.previewPath,
embedPath: v.embedPath,
views: v.views,
viewers: v.viewers,
likes: v.likes,
dislikes: v.dislikes,
isLive: v.isLive || false,
nsfw: v.nsfw,
host: v.host,
url: v.url,
files: v.files,
streamingPlaylists: v.streamingPlaylists,
tags: (v as IndexableVideoDetails).tags ? (v as IndexableVideoDetails).tags : undefined,
account: formatActorForDB(v.account),
channel: formatActorForDB(v.channel)
}
if (isVideoDetails(v)) {
return {
...video,
trackerUrls: v.trackerUrls,
descriptionPath: v.descriptionPath,
support: v.support,
commentsEnabled: v.commentsEnabled,
downloadEnabled: v.downloadEnabled
}
}
return video
}
export function formatVideoForAPI (v: DBVideoDetails, fromHost?: string): APIVideo {
return {
id: v.id,
uuid: v.uuid,
shortUUID: v.shortUUID,
score: v._rankingScore,
createdAt: new Date(v.createdAt),
updatedAt: new Date(v.updatedAt),
publishedAt: new Date(v.publishedAt),
originallyPublishedAt: new Date(v.originallyPublishedAt),
category: {
id: v.category.id,
label: v.category.label
},
licence: {
id: v.licence.id,
label: v.licence.label
},
language: {
id: v.language.id,
label: v.language.label
},
privacy: {
id: v.privacy.id,
label: v.privacy.label
},
name: v.name,
description: v.description,
truncatedDescription: v.truncatedDescription,
duration: v.duration,
tags: v.tags,
thumbnailPath: v.thumbnailPath,
thumbnailUrl: buildUrl(v.host, v.thumbnailPath),
previewPath: v.previewPath,
previewUrl: buildUrl(v.host, v.previewPath),
embedPath: v.embedPath,
embedUrl: buildUrl(v.host, v.embedPath),
url: v.url,
isLocal: fromHost && fromHost === v.host,
views: v.views,
viewers: v.viewers,
likes: v.likes,
dislikes: v.dislikes,
isLive: v.isLive,
nsfw: v.nsfw,
account: formatActorSummaryForAPI(v.account),
channel: formatActorSummaryForAPI(v.channel)
}
}
// ---------------------------------------------------------------------------
export function isVideoDetails (video: IndexableVideo | IndexableVideoDetails): video is IndexableVideoDetails {
return (video as IndexableVideoDetails).commentsEnabled !== undefined
}

3
server/lib/meilisearch/shared/index.ts Normal file
View File

@ -0,0 +1,3 @@
export * from './meilisearch-actor'
export * from './meilisearch-avatar'
export * from './query-helpers'

32
server/lib/meilisearch/shared/meilisearch-actor.ts Normal file
View File

@ -0,0 +1,32 @@
import { AccountSummary, VideoChannelSummary } from '@peertube/peertube-types'
import { formatActorImageForAPI, formatActorImageForDB, formatActorImagesForAPI, formatActorImagesForDB } from './meilisearch-avatar'
import { DBAccountSummary } from '../../../types/account.model'
import { DBChannelSummary } from '../../../types/channel.model'
export function formatActorSummaryForAPI (actor: DBAccountSummary | DBChannelSummary) {
return {
id: actor.id,
name: actor.name,
displayName: actor.displayName,
url: actor.url,
host: actor.host,
avatar: formatActorImageForAPI(actor.avatar),
avatars: formatActorImagesForAPI(actor.avatars, actor.avatar)
}
}
export function formatActorForDB (actor: AccountSummary | VideoChannelSummary) {
return {
id: actor.id,
name: actor.name,
displayName: actor.displayName,
url: actor.url,
host: actor.host,
handle: `${actor.name}@${actor.host}`,
avatar: formatActorImageForDB(actor.avatar, actor.host),
avatars: formatActorImagesForDB(actor.avatars, actor.host)
}
}

46
server/lib/meilisearch/shared/meilisearch-avatar.ts Normal file
View File

@ -0,0 +1,46 @@
import { ActorImage } from '@peertube/peertube-types'
import { buildUrl } from '../../../helpers/utils'
import { DBActorImage } from '../../../types/actor.model'
export function formatActorImageForAPI (image?: DBActorImage) {
if (!image) return null
return {
url: image.url,
path: image.path,
width: image.width,
createdAt: new Date(image.createdAt),
updatedAt: new Date(image.updatedAt)
}
}
export function formatActorImagesForAPI (images?: DBActorImage[], image?: DBActorImage) {
// Does not exist in PeerTube < 4.2
if (!images) {
if (!image) return []
return [ formatActorImageForAPI(image) ]
}
return images.map(a => formatActorImageForAPI(a))
}
// ---------------------------------------------------------------------------
export function formatActorImageForDB (image: ActorImage, host: string) {
if (!image) return null
return {
url: buildUrl(host, image.path),
path: image.path,
width: image.width,
createdAt: new Date(image.createdAt).getTime(),
updatedAt: new Date(image.updatedAt).getTime()
}
}
export function formatActorImagesForDB (images: ActorImage[], host: string) {
if (!images) return null
return images.map(image => formatActorImageForDB(image, host))
}

22
server/lib/meilisearch/shared/query-helpers.ts Normal file
View File

@ -0,0 +1,22 @@
import validator from 'validator'
import { buildInQuery } from '../meilisearch-queries'
export function addUUIDFilters (filters: string[], uuids: string[]) {
if (!filters || filters.length === 0) return
const result = {
shortUUIDs: [] as string[],
uuids: [] as string[]
}
for (const uuid of uuids) {
if (validator.isUUID(uuid)) result.uuids.push(uuid)
else result.shortUUIDs.push(uuid)
}
const parts: string[] = []
if (result.uuids.length !== 0) parts.push(buildInQuery('uuid', result.uuids))
if (result.shortUUIDs.length !== 0) parts.push(buildInQuery('shortUUID', result.shortUUIDs))
filters.push(parts.join(' OR '))
}

5
server/lib/requests/peertube-instance.ts
View File

@ -1,10 +1,10 @@
import { IndexablePlaylist } from 'server/types/playlist.model'
import { ResultList, Video, VideoChannel, VideoDetails, VideoPlaylist, VideosCommonQuery } from '@peertube/peertube-types' import { ResultList, Video, VideoChannel, VideoDetails, VideoPlaylist, VideosCommonQuery } from '@peertube/peertube-types'
import { doRequestWithRetries } from '../../helpers/requests' import { doRequestWithRetries } from '../../helpers/requests'
import { INDEXER_COUNT, REQUESTS } from '../../initializers/constants' import { INDEXER_COUNT, REQUESTS } from '../../initializers/constants'
import { IndexableChannel } from '../../types/channel.model' import { IndexableChannel } from '../../types/channel.model'
import { IndexableDoc } from '../../types/indexable-doc.model' import { IndexableDoc } from '../../types/indexable-doc.model'
import { IndexableVideo } from '../../types/video.model' import { IndexableVideo } from '../../types/video.model'
import { IndexablePlaylist } from '../../types/playlist.model'
async function getVideo (host: string, uuid: string): Promise<IndexableVideo> { async function getVideo (host: string, uuid: string): Promise<IndexableVideo> {
const url = 'https://' + host + '/api/v1/videos/' + uuid const url = 'https://' + host + '/api/v1/videos/' + uuid
@ -92,7 +92,6 @@ async function getPlaylistsOf (host: string, handle: string, start: number): Pro
function prepareVideoForDB <T extends Video> (video: T, host: string): T & IndexableDoc { function prepareVideoForDB <T extends Video> (video: T, host: string): T & IndexableDoc {
return Object.assign(video, { return Object.assign(video, {
elasticSearchId: host + video.id,
host, host,
url: 'https://' + host + '/videos/watch/' + video.uuid url: 'https://' + host + '/videos/watch/' + video.uuid
}) })
@ -100,7 +99,6 @@ function prepareVideoForDB <T extends Video> (video: T, host: string): T & Index
function prepareChannelForDB (channel: VideoChannel, host: string, videosCount: number): IndexableChannel { function prepareChannelForDB (channel: VideoChannel, host: string, videosCount: number): IndexableChannel {
return Object.assign(channel, { return Object.assign(channel, {
elasticSearchId: host + channel.id,
host, host,
videosCount, videosCount,
url: 'https://' + host + '/video-channels/' + channel.name url: 'https://' + host + '/video-channels/' + channel.name
@ -109,7 +107,6 @@ function prepareChannelForDB (channel: VideoChannel, host: string, videosCount:
function preparePlaylistForDB (playlist: VideoPlaylist, host: string): IndexablePlaylist { function preparePlaylistForDB (playlist: VideoPlaylist, host: string): IndexablePlaylist {
return Object.assign(playlist, { return Object.assign(playlist, {
elasticSearchId: host + playlist.id,
host, host,
url: 'https://' + host + '/videos/watch/playlist/' + playlist.uuid url: 'https://' + host + '/videos/watch/playlist/' + playlist.uuid
}) })

37
server/lib/schedulers/indexation-scheduler.ts
View File

@ -1,15 +1,17 @@
import Bluebird from 'bluebird' import Bluebird from 'bluebird'
import { IndexablePlaylist } from 'server/types/playlist.model'
import { inspect } from 'util' import { inspect } from 'util'
import { logger } from '../../helpers/logger' import { logger } from '../../helpers/logger'
import { INDEXER_HOST_CONCURRENCY, INDEXER_COUNT, INDEXER_LIMIT, SCHEDULER_INTERVALS_MS } from '../../initializers/constants' import { INDEXER_HOST_CONCURRENCY, INDEXER_COUNT, INDEXER_LIMIT, SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
import { IndexableVideo } from '../../types/video.model' import { IndexableVideo } from '../../types/video.model'
import { buildInstanceHosts } from '../elastic-search/elastic-search-instances' import { buildInstanceHosts } from '../meilisearch/meilisearch-instances'
import { ChannelIndexer } from '../indexers/channel-indexer' import { ChannelIndexer } from '../indexers/channel-indexer'
import { PlaylistIndexer } from '../indexers/playlist-indexer' import { PlaylistIndexer } from '../indexers/playlist-indexer'
import { VideoIndexer } from '../indexers/video-indexer' import { VideoIndexer } from '../indexers/video-indexer'
import { getPlaylistsOf, getVideos } from '../requests/peertube-instance' import { getPlaylistsOf, getVideos } from '../requests/peertube-instance'
import { AbstractScheduler } from './abstract-scheduler' import { AbstractScheduler } from './abstract-scheduler'
import { IndexablePlaylist } from '../../types/playlist.model'
import { buildDBChannelPrimaryKey } from '../meilisearch/meilisearch-channels'
import { getVideosUpdatedAt } from '../meilisearch/meilisearch-videos'
export class IndexationScheduler extends AbstractScheduler { export class IndexationScheduler extends AbstractScheduler {
@ -70,17 +72,13 @@ export class IndexationScheduler extends AbstractScheduler {
} }
}, { concurrency: INDEXER_HOST_CONCURRENCY }) }, { concurrency: INDEXER_HOST_CONCURRENCY })
for (const o of this.indexers) {
await o.refreshIndex()
}
logger.info('Indexer ended.') logger.info('Indexer ended.')
} }
private async indexHost (host: string) { private async indexHost (host: string) {
const channelsToSync = new Set<string>() const channelsToSync = new Set<string>()
const existingChannelsId = new Set<number>() const existingChannelsId = new Set<string>()
const existingVideosId = new Set<number>() const existingVideosId = new Set<string>()
let videos: IndexableVideo[] = [] let videos: IndexableVideo[] = []
let start = 0 let start = 0
@ -96,21 +94,27 @@ export class IndexationScheduler extends AbstractScheduler {
start += videos.length start += videos.length
if (videos.length !== 0) { if (videos.length !== 0) {
const { created } = await this.videoIndexer.indexElements(videos) const videosFromDB = await getVideosUpdatedAt(videos.map(v => v.uuid))
await this.videoIndexer.indexElements(videos)
logger.debug('Indexed %d videos from %s.', videos.length, host) logger.debug('Indexed %d videos from %s.', videos.length, host)
// Fetch complete video foreach created video (to get tags) // Fetch complete video foreach created video (to get tags) if needed
for (const c of created) { for (const video of videos) {
this.videoIndexer.scheduleIndexation(host, c.uuid) const videoDB = videosFromDB.find(v => v.uuid === video.uuid)
if (!videoDB || videoDB.updatedAt !== new Date(video.updatedAt).getTime()) {
this.videoIndexer.scheduleIndexation(host, video.uuid)
}
} }
} }
for (const video of videos) { for (const video of videos) {
channelsToSync.add(video.channel.name) channelsToSync.add(video.channel.name)
existingChannelsId.add(video.channel.id) existingChannelsId.add(buildDBChannelPrimaryKey(video.channel))
existingVideosId.add(video.id) existingVideosId.add(video.uuid)
} }
} while (videos.length === INDEXER_COUNT && start < INDEXER_LIMIT) } while (videos.length === INDEXER_COUNT && start < INDEXER_LIMIT)
@ -120,6 +124,7 @@ export class IndexationScheduler extends AbstractScheduler {
this.channelIndexer.scheduleIndexation(host, c) this.channelIndexer.scheduleIndexation(host, c)
} }
logger.info('Removing non-existing channels and videos from ' + host)
await this.channelIndexer.removeNotExisting(host, existingChannelsId) await this.channelIndexer.removeNotExisting(host, existingChannelsId)
await this.videoIndexer.removeNotExisting(host, existingVideosId) await this.videoIndexer.removeNotExisting(host, existingVideosId)
@ -127,7 +132,7 @@ export class IndexationScheduler extends AbstractScheduler {
} }
private async indexPlaylists (host: string, channelHandles: string[]) { private async indexPlaylists (host: string, channelHandles: string[]) {
const existingPlaylistsId = new Set<number>() const existingPlaylistsId = new Set<string>()
logger.info('Adding playlist data from %s.', host) logger.info('Adding playlist data from %s.', host)
@ -150,7 +155,7 @@ export class IndexationScheduler extends AbstractScheduler {
} }
for (const playlist of playlists) { for (const playlist of playlists) {
existingPlaylistsId.add(playlist.id) existingPlaylistsId.add(playlist.uuid)
} }
} while (playlists.length === INDEXER_COUNT && start < INDEXER_LIMIT) } while (playlists.length === INDEXER_COUNT && start < INDEXER_LIMIT)
} }

25
server/types/account.model.ts Normal file
View File

@ -0,0 +1,25 @@
import { Account, AccountSummary } from '@peertube/peertube-types'
import { DBActorImage } from './actor.model'
type IgnoredAccountFields = 'createdAt' | 'updatedAt' | 'avatar' | 'avatars'
interface DBAccountShared {
handle: string
avatar: DBActorImage
avatars: DBActorImage[]
}
export interface DBAccount extends Omit<Account, IgnoredAccountFields> {
handle: string
avatar: DBActorImage
avatars: DBActorImage[]
createdAt: number
updatedAt: number
}
export interface DBAccountSummary extends DBAccountShared, Omit<AccountSummary, 'avatar' | 'avatars'> {
}

10
server/types/actor.model.ts
View File

@ -1,11 +1,7 @@
import { ActorImage } from '@peertube/peertube-types' import { ActorImage } from '@peertube/peertube-types'
export type AdditionalActorAttributes = { export type DBActorImage = Omit<ActorImage, 'createdAt' | 'updatedAt'> & {
handle: string
url: string url: string
createdAt: number
avatar: ActorImageExtended updatedAt: number
avatars: ActorImageExtended[]
} }
export type ActorImageExtended = ActorImage & { url: string }

45
server/types/channel.model.ts
View File

@ -1,33 +1,42 @@
import { Account, VideoChannel, VideoChannelSummary } from '@peertube/peertube-types' import { VideoChannel, VideoChannelSummary } from '@peertube/peertube-types'
import { ActorImageExtended, AdditionalActorAttributes } from './actor.model' import { DBActorImage } from './actor.model'
import { IndexableDoc } from './indexable-doc.model' import { IndexableDoc } from './indexable-doc.model'
import { DBAccount } from './account.model'
export interface IndexableChannel extends VideoChannel, IndexableDoc { export interface IndexableChannel extends VideoChannel, IndexableDoc {
url: string
} }
export interface DBChannel extends Omit<VideoChannel, 'isLocal'> { interface DBChannelShared {
indexedAt: Date
handle: string handle: string
url: string
ownerAccount?: Account & AdditionalActorAttributes avatar: DBActorImage
avatars: DBActorImage[]
avatar: ActorImageExtended
avatars: ActorImageExtended[]
banner: ActorImageExtended
banners: ActorImageExtended[]
score?: number
} }
export interface DBChannelSummary extends VideoChannelSummary { type IgnoredChannelFields = 'ownerAccount' | 'isLocal' | 'createdAt' | 'updatedAt' | 'avatar' | 'avatars' | 'banner' | 'banners'
indexedAt: Date
export interface DBChannel extends DBChannelShared, Omit<VideoChannel, IgnoredChannelFields> {
indexedAt: number
primaryKey: string
ownerAccount?: DBAccount
banner: DBActorImage
banners: DBActorImage[]
createdAt: number
updatedAt: number
_rankingScore?: number
}
export interface DBChannelSummary extends DBChannelShared, Omit<VideoChannelSummary, 'avatar' | 'avatars'> {
} }
// Results from the search API // Results from the search API
export interface EnhancedVideoChannel extends VideoChannel { export interface APIVideoChannel extends VideoChannel {
videosCount: number videosCount: number
score: number score: number

1
server/types/indexable-doc.model.ts
View File

@ -1,5 +1,4 @@
export interface IndexableDoc { export interface IndexableDoc {
elasticSearchId: string
host: string host: string
url: string url: string
} }

20
server/types/playlist.model.ts
View File

@ -1,24 +1,26 @@
import { AccountSummary, VideoChannelSummary, VideoPlaylist } from '@peertube/peertube-types' import { VideoPlaylist } from '@peertube/peertube-types'
import { AdditionalActorAttributes } from './actor.model'
import { IndexableDoc } from './indexable-doc.model' import { IndexableDoc } from './indexable-doc.model'
import { DBAccountSummary } from './account.model'
import { DBChannelSummary } from './channel.model'
export interface IndexablePlaylist extends VideoPlaylist, IndexableDoc { export interface IndexablePlaylist extends VideoPlaylist, IndexableDoc {
url: string
} }
export interface DBPlaylist extends Omit<VideoPlaylist, 'isLocal'> { export interface DBPlaylist extends Omit<VideoPlaylist, 'isLocal' | 'createdAt' | 'updatedAt' | 'ownerAccount' | 'videoChannel'> {
indexedAt: Date indexedAt: number
createdAt: number
updatedAt: number
host: string host: string
// Added by the query // Added by the query
score?: number _rankingScore?: number
ownerAccount: AccountSummary & AdditionalActorAttributes ownerAccount: DBAccountSummary
videoChannel: VideoChannelSummary & AdditionalActorAttributes videoChannel: DBChannelSummary
} }
// Results from the search API // Results from the search API
export interface EnhancedPlaylist extends VideoPlaylist { export interface APIPlaylist extends VideoPlaylist {
score: number score: number
} }

39
server/types/video.model.ts
View File

@ -1,6 +1,7 @@
import { Account, AccountSummary, Video, VideoChannel, VideoChannelSummary, VideoDetails } from '@peertube/peertube-types' import { Video, VideoDetails } from '@peertube/peertube-types'
import { AdditionalActorAttributes } from './actor.model'
import { IndexableDoc } from './indexable-doc.model' import { IndexableDoc } from './indexable-doc.model'
import { DBChannel, DBChannelSummary } from './channel.model'
import { DBAccount, DBAccountSummary } from './account.model'
export interface IndexableVideo extends Video, IndexableDoc { export interface IndexableVideo extends Video, IndexableDoc {
} }
@ -8,28 +9,34 @@ export interface IndexableVideo extends Video, IndexableDoc {
export interface IndexableVideoDetails extends VideoDetails, IndexableDoc { export interface IndexableVideoDetails extends VideoDetails, IndexableDoc {
} }
export interface DBVideoDetails extends Omit<VideoDetails, 'isLocal'> { type IgnoredVideoFields = 'isLocal' | 'createdAt' | 'updatedAt' | 'publishedAt' | 'originallyPublishedAt' | 'channel' | 'account'
indexedAt: Date
interface DBVideoShared {
indexedAt: number
createdAt: number
updatedAt: number
publishedAt: number
originallyPublishedAt: number
host: string host: string
url: string url: string
account: Account & AdditionalActorAttributes
channel: VideoChannel & AdditionalActorAttributes
score?: number
} }
export interface DBVideo extends Omit<Video, 'isLocal'> { export interface DBVideo extends Omit<Video, IgnoredVideoFields>, DBVideoShared {
indexedAt: Date account: DBAccountSummary
host: string channel: DBChannelSummary
url: string }
account: AccountSummary & AdditionalActorAttributes export interface DBVideoDetails extends Omit<VideoDetails, IgnoredVideoFields>, DBVideoShared {
channel: VideoChannelSummary & AdditionalActorAttributes account: DBAccount
channel: DBChannel
_rankingScore?: number
} }
// Results from the search API // Results from the search API
export interface EnhancedVideo extends Video { export interface APIVideo extends Video {
tags: VideoDetails['tags'] tags: VideoDetails['tags']
score: number score: number

6
tsconfig.json
View File

@ -16,13 +16,11 @@
], ],
"types": [ "types": [
"node" "node"
], ]
"baseUrl": "."
}, },
"exclude": [ "exclude": [
"node_modules", "node_modules",
"dist", "dist",
"client", "client"
"PeerTube"
] ]
} }

93
yarn.lock
View File

@ -980,26 +980,6 @@
enabled "2.0.x" enabled "2.0.x"
kuler "^2.0.0" kuler "^2.0.0"
"@elastic/elasticsearch@^8.2.1":
version "8.5.0"
resolved "https://registry.yarnpkg.com/@elastic/elasticsearch/-/elasticsearch-8.5.0.tgz#407aee0950a082ee76735a567f2571cf4301d4ea"
integrity sha512-iOgr/3zQi84WmPhAplnK2W13R89VXD2oc6WhlQmH3bARQwmI+De23ZJKBEn7bvuG/AHMAqasPXX7uJIiJa2MqQ==
dependencies:
"@elastic/transport" "^8.2.0"
tslib "^2.4.0"
"@elastic/transport@^8.2.0":
version "8.3.1"
resolved "https://registry.yarnpkg.com/@elastic/transport/-/transport-8.3.1.tgz#e7569d7df35b03108ea7aa886113800245faa17f"
integrity sha512-jv/Yp2VLvv5tSMEOF8iGrtL2YsYHbpf4s+nDsItxUTLFTzuJGpnsB/xBlfsoT2kAYEnWHiSJuqrbRcpXEI/SEQ==
dependencies:
debug "^4.3.4"
hpagent "^1.0.0"
ms "^2.1.3"
secure-json-parse "^2.4.0"
tslib "^2.4.0"
undici "^5.5.1"
"@eslint/eslintrc@^1.3.3": "@eslint/eslintrc@^1.3.3":
version "1.3.3" version "1.3.3"
resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-1.3.3.tgz#2b044ab39fdfa75b4688184f9e573ce3c5b0ff95" resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-1.3.3.tgz#2b044ab39fdfa75b4688184f9e573ce3c5b0ff95"
@ -1202,7 +1182,7 @@
resolved "https://registry.yarnpkg.com/@types/async/-/async-3.2.16.tgz#7473aa015cf8a60584a94dc79b9203d465c32b41" resolved "https://registry.yarnpkg.com/@types/async/-/async-3.2.16.tgz#7473aa015cf8a60584a94dc79b9203d465c32b41"
integrity sha512-jnlGp5Z/cAZ7JVYyLnSDuYJ+YyYm0o2yzL8Odv6ckWmGMow3j/P/wgfziybB044cXXA93lEuymJyxVR8Iz2amQ== integrity sha512-jnlGp5Z/cAZ7JVYyLnSDuYJ+YyYm0o2yzL8Odv6ckWmGMow3j/P/wgfziybB044cXXA93lEuymJyxVR8Iz2amQ==
"@types/bluebird@*", "@types/bluebird@^3.5.33": "@types/bluebird@^3.5.33":
version "3.5.38" version "3.5.38"
resolved "https://registry.yarnpkg.com/@types/bluebird/-/bluebird-3.5.38.tgz#7a671e66750ccd21c9fc9d264d0e1e5330bc9908" resolved "https://registry.yarnpkg.com/@types/bluebird/-/bluebird-3.5.38.tgz#7a671e66750ccd21c9fc9d264d0e1e5330bc9908"
integrity sha512-yR/Kxc0dd4FfwtEoLZMoqJbM/VE/W7hXn/MIjb+axcwag0iFmSPK7OBUZq1YWLynJUoWQkfUrI7T0HDqGApNSg== integrity sha512-yR/Kxc0dd4FfwtEoLZMoqJbM/VE/W7hXn/MIjb+axcwag0iFmSPK7OBUZq1YWLynJUoWQkfUrI7T0HDqGApNSg==
@ -1242,13 +1222,6 @@
dependencies: dependencies:
"@types/node" "*" "@types/node" "*"
"@types/continuation-local-storage@*":
version "3.2.4"
resolved "https://registry.yarnpkg.com/@types/continuation-local-storage/-/continuation-local-storage-3.2.4.tgz#655c8ffd9327aa60fb8ae773a5f2efbc973a7cbb"
integrity sha512-OT32vCVMymU1JMPKDeY0lX3cduAr0Pm/VwIbxygMeDS4lRcv57qYXn9bMwBRcRnEpXKBb/r4xYaZCARTZllP0A==
dependencies:
"@types/node" "*"
"@types/cookie@^0.4.1": "@types/cookie@^0.4.1":
version "0.4.1" version "0.4.1"
resolved "https://registry.yarnpkg.com/@types/cookie/-/cookie-0.4.1.tgz#bfd02c1f2224567676c1545199f87c3a861d878d" resolved "https://registry.yarnpkg.com/@types/cookie/-/cookie-0.4.1.tgz#bfd02c1f2224567676c1545199f87c3a861d878d"
@ -1333,7 +1306,7 @@
dependencies: dependencies:
"@types/node" "*" "@types/node" "*"
"@types/lodash@*", "@types/lodash@^4.14.182": "@types/lodash@^4.14.182":
version "4.14.191" version "4.14.191"
resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.191.tgz#09511e7f7cba275acd8b419ddac8da9a6a79e2fa" resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.191.tgz#09511e7f7cba275acd8b419ddac8da9a6a79e2fa"
integrity sha512-BdZ5BCCvho3EIXw6wUCXHe7rS53AIDPLE+JzwgT+OsJk53oBfbSmZZ7CX4VaRoN78N+TJpFi9QPlfIVNmJYWxQ== integrity sha512-BdZ5BCCvho3EIXw6wUCXHe7rS53AIDPLE+JzwgT+OsJk53oBfbSmZZ7CX4VaRoN78N+TJpFi9QPlfIVNmJYWxQ==
@ -1428,16 +1401,6 @@
resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.3.13.tgz#da4bfd73f49bd541d28920ab0e2bf0ee80f71c91" resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.3.13.tgz#da4bfd73f49bd541d28920ab0e2bf0ee80f71c91"
integrity sha512-21cFJr9z3g5dW8B0CVI9g2O9beqaThGQ6ZFBqHfwhzLDKUxaqTIy3vnfah/UPkfOiF2pLq+tGz+W8RyCskuslw== integrity sha512-21cFJr9z3g5dW8B0CVI9g2O9beqaThGQ6ZFBqHfwhzLDKUxaqTIy3vnfah/UPkfOiF2pLq+tGz+W8RyCskuslw==
"@types/sequelize@^4.28.13":
version "4.28.14"
resolved "https://registry.yarnpkg.com/@types/sequelize/-/sequelize-4.28.14.tgz#70302689ddef09f5d472b548a14e5a150e690aff"
integrity sha512-O8lTJ8YPVVaoY9xjduchDlo0MOS3w262pro2H1QMuFIo/kc/p1elP/UxLOTP2wcVO2cFd6Gvghg9ZSAiJi0GLA==
dependencies:
"@types/bluebird" "*"
"@types/continuation-local-storage" "*"
"@types/lodash" "*"
"@types/validator" "*"
"@types/serve-static@*": "@types/serve-static@*":
version "1.15.0" version "1.15.0"
resolved "https://registry.yarnpkg.com/@types/serve-static/-/serve-static-1.15.0.tgz#c7930ff61afb334e121a9da780aac0d9b8f34155" resolved "https://registry.yarnpkg.com/@types/serve-static/-/serve-static-1.15.0.tgz#c7930ff61afb334e121a9da780aac0d9b8f34155"
@ -1451,7 +1414,7 @@
resolved "https://registry.yarnpkg.com/@types/tough-cookie/-/tough-cookie-4.0.2.tgz#6286b4c7228d58ab7866d19716f3696e03a09397" resolved "https://registry.yarnpkg.com/@types/tough-cookie/-/tough-cookie-4.0.2.tgz#6286b4c7228d58ab7866d19716f3696e03a09397"
integrity sha512-Q5vtl1W5ue16D+nIaW8JWebSSraJVlK+EthKn7e7UcD4KWsaSJ8BqGPXNaPghgtcn/fhvrN17Tv8ksUsQpiplw== integrity sha512-Q5vtl1W5ue16D+nIaW8JWebSSraJVlK+EthKn7e7UcD4KWsaSJ8BqGPXNaPghgtcn/fhvrN17Tv8ksUsQpiplw==
"@types/validator@*", "@types/validator@^13.0.0", "@types/validator@^13.7.1", "@types/validator@^13.7.2": "@types/validator@^13.0.0", "@types/validator@^13.7.1", "@types/validator@^13.7.2":
version "13.7.10" version "13.7.10"
resolved "https://registry.yarnpkg.com/@types/validator/-/validator-13.7.10.tgz#f9763dc0933f8324920afa9c0790308eedf55ca7" resolved "https://registry.yarnpkg.com/@types/validator/-/validator-13.7.10.tgz#f9763dc0933f8324920afa9c0790308eedf55ca7"
integrity sha512-t1yxFAR2n0+VO6hd/FJ9F2uezAZVWHLmpmlJzm1eX03+H7+HsuTAp7L8QJs+2pQCfWkP1+EXsGK9Z9v7o/qPVQ== integrity sha512-t1yxFAR2n0+VO6hd/FJ9F2uezAZVWHLmpmlJzm1eX03+H7+HsuTAp7L8QJs+2pQCfWkP1+EXsGK9Z9v7o/qPVQ==
@ -1824,7 +1787,7 @@ bullmq@^1.87.0:
tslib "^2.0.0" tslib "^2.0.0"
uuid "^9.0.0" uuid "^9.0.0"
busboy@^1.0.0, busboy@^1.6.0: busboy@^1.0.0:
version "1.6.0" version "1.6.0"
resolved "https://registry.yarnpkg.com/busboy/-/busboy-1.6.0.tgz#966ea36a9502e43cdb9146962523b92f531f6893" resolved "https://registry.yarnpkg.com/busboy/-/busboy-1.6.0.tgz#966ea36a9502e43cdb9146962523b92f531f6893"
integrity sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA== integrity sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==
@ -2039,6 +2002,13 @@ cron-parser@^4.6.0:
dependencies: dependencies:
luxon "^3.1.0" luxon "^3.1.0"
cross-fetch@^3.1.6:
version "3.1.8"
resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-3.1.8.tgz#0327eba65fd68a7d119f8fb2bf9334a1a7956f82"
integrity sha512-cvA+JwZoU0Xq+h6WkMvAUqPEYy92Obet6UdKLfW60qn99ftItKjB5T+BkyWOFWe2pUyfQ+IJHmpOTznqk1M6Kg==
dependencies:
node-fetch "^2.6.12"
cross-spawn@^7.0.2, cross-spawn@^7.0.3: cross-spawn@^7.0.2, cross-spawn@^7.0.3:
version "7.0.3" version "7.0.3"
resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6" resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6"
@ -3496,6 +3466,13 @@ media-typer@0.3.0:
resolved "https://registry.yarnpkg.com/media-typer/-/media-typer-0.3.0.tgz#8710d7af0aa626f8fffa1ce00168545263255748" resolved "https://registry.yarnpkg.com/media-typer/-/media-typer-0.3.0.tgz#8710d7af0aa626f8fffa1ce00168545263255748"
integrity sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ== integrity sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==
meilisearch@^0.35.0:
version "0.35.0"
resolved "https://registry.yarnpkg.com/meilisearch/-/meilisearch-0.35.0.tgz#c94d5f266e39ad5ed5a666782e729805a4e8a956"
integrity sha512-gF1I6K5/Wpe7BWfjBnG+o19y/FIpJ9HbN+byON6CB9U3uE7qc6GvwUbjKOllh7LKXQVVxH/kCu7Jn0ODCUwqbQ==
dependencies:
cross-fetch "^3.1.6"
memoizee@^0.4.14: memoizee@^0.4.14:
version "0.4.15" version "0.4.15"
resolved "https://registry.yarnpkg.com/memoizee/-/memoizee-0.4.15.tgz#e6f3d2da863f318d02225391829a6c5956555b72" resolved "https://registry.yarnpkg.com/memoizee/-/memoizee-0.4.15.tgz#e6f3d2da863f318d02225391829a6c5956555b72"
@ -3634,7 +3611,7 @@ ms@2.1.2:
resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009"
integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==
ms@2.1.3, ms@^2.1.1, ms@^2.1.3: ms@2.1.3, ms@^2.1.1:
version "2.1.3" version "2.1.3"
resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2"
integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==
@ -3702,6 +3679,13 @@ next-tick@1, next-tick@^1.1.0:
resolved "https://registry.yarnpkg.com/next-tick/-/next-tick-1.1.0.tgz#1836ee30ad56d67ef281b22bd199f709449b35eb" resolved "https://registry.yarnpkg.com/next-tick/-/next-tick-1.1.0.tgz#1836ee30ad56d67ef281b22bd199f709449b35eb"
integrity sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ== integrity sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ==
node-fetch@^2.6.12:
version "2.7.0"
resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d"
integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==
dependencies:
whatwg-url "^5.0.0"
node-gyp-build-optional-packages@5.0.3: node-gyp-build-optional-packages@5.0.3:
version "5.0.3" version "5.0.3"
resolved "https://registry.yarnpkg.com/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.3.tgz#92a89d400352c44ad3975010368072b41ad66c17" resolved "https://registry.yarnpkg.com/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.3.tgz#92a89d400352c44ad3975010368072b41ad66c17"
@ -4686,6 +4670,11 @@ tough-cookie@~2.5.0:
psl "^1.1.28" psl "^1.1.28"
punycode "^2.1.1" punycode "^2.1.1"
tr46@~0.0.3:
version "0.0.3"
resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a"
integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==
triple-beam@^1.3.0: triple-beam@^1.3.0:
version "1.3.0" version "1.3.0"
resolved "https://registry.yarnpkg.com/triple-beam/-/triple-beam-1.3.0.tgz#a595214c7298db8339eeeee083e4d10bd8cb8dd9" resolved "https://registry.yarnpkg.com/triple-beam/-/triple-beam-1.3.0.tgz#a595214c7298db8339eeeee083e4d10bd8cb8dd9"
@ -4787,13 +4776,6 @@ unbox-primitive@^1.0.2:
has-symbols "^1.0.3" has-symbols "^1.0.3"
which-boxed-primitive "^1.0.2" which-boxed-primitive "^1.0.2"
undici@^5.5.1:
version "5.14.0"
resolved "https://registry.yarnpkg.com/undici/-/undici-5.14.0.tgz#1169d0cdee06a4ffdd30810f6228d57998884d00"
integrity sha512-yJlHYw6yXPPsuOH0x2Ib1Km61vu4hLiRRQoafs+WUgX1vO64vgnxiCEN9dpIrhZyHFsai3F0AEj4P9zy19enEQ==
dependencies:
busboy "^1.6.0"
universalify@^2.0.0: universalify@^2.0.0:
version "2.0.0" version "2.0.0"
resolved "https://registry.yarnpkg.com/universalify/-/universalify-2.0.0.tgz#75a4984efedc4b08975c5aeb73f530d02df25717" resolved "https://registry.yarnpkg.com/universalify/-/universalify-2.0.0.tgz#75a4984efedc4b08975c5aeb73f530d02df25717"
@ -4855,6 +4837,19 @@ verror@1.10.0:
core-util-is "1.0.2" core-util-is "1.0.2"
extsprintf "^1.2.0" extsprintf "^1.2.0"
webidl-conversions@^3.0.0:
version "3.0.1"
resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871"
integrity sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==
whatwg-url@^5.0.0:
version "5.0.0"
resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d"
integrity sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==
dependencies:
tr46 "~0.0.3"
webidl-conversions "^3.0.0"
which-boxed-primitive@^1.0.2: which-boxed-primitive@^1.0.2:
version "1.0.2" version "1.0.2"
resolved "https://registry.yarnpkg.com/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz#13757bc89b209b049fe5d86430e21cf40a89a8e6" resolved "https://registry.yarnpkg.com/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz#13757bc89b209b049fe5d86430e21cf40a89a8e6"