From 7fb1b791ae70530fde8b0765117d3e3b56d10dfa Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 9 Nov 2023 14:34:36 +0100 Subject: [PATCH] Move from elastic search to meilisearch --- README.md | 20 +- client/src/components/ChannelResult.vue | 2 +- config/default.yaml | 98 +--- config/test.yaml | 7 +- package.json | 3 +- server/controllers/api/search-channels.ts | 2 +- server/controllers/api/search-playlists.ts | 2 +- server/controllers/api/search-videos.ts | 2 +- server/helpers/elastic-search.ts | 30 - server/helpers/meilisearch.ts | 9 + server/initializers/constants.ts | 154 +---- .../elastic-search/elastic-search-channels.ts | 209 ------- .../elastic-search/elastic-search-index.ts | 91 --- .../elastic-search-playlists.ts | 234 -------- .../elastic-search/elastic-search-queries.ts | 134 ----- .../elastic-search/elastic-search-videos.ts | 528 ------------------ .../shared/elastic-search-actor.ts | 104 ---- .../shared/elastic-search-avatar.ts | 77 --- server/lib/elastic-search/shared/index.ts | 3 - .../elastic-search/shared/query-helpers.ts | 88 --- server/lib/indexers/channel-indexer.ts | 23 +- server/lib/indexers/playlist-indexer.ts | 15 +- .../lib/indexers/shared/abstract-indexer.ts | 62 +- server/lib/indexers/video-indexer.ts | 56 +- .../lib/meilisearch/meilisearch-channels.ts | 141 +++++ .../meilisearch-instances.ts} | 27 +- .../lib/meilisearch/meilisearch-playlists.ts | 108 ++++ server/lib/meilisearch/meilisearch-queries.ts | 27 + server/lib/meilisearch/meilisearch-videos.ts | 226 ++++++++ server/lib/meilisearch/shared/index.ts | 3 + .../meilisearch/shared/meilisearch-actor.ts | 32 ++ .../meilisearch/shared/meilisearch-avatar.ts | 46 ++ .../lib/meilisearch/shared/query-helpers.ts | 22 + server/lib/requests/peertube-instance.ts | 5 +- server/lib/schedulers/indexation-scheduler.ts | 37 +- server/types/account.model.ts | 25 + server/types/actor.model.ts | 10 +- server/types/channel.model.ts | 45 +- server/types/indexable-doc.model.ts | 1 - server/types/playlist.model.ts | 20 +- server/types/video.model.ts | 39 +- tsconfig.json | 6 +- yarn.lock | 93 ++- 43 files changed, 908 insertions(+), 1958 deletions(-) delete mode 100644 server/helpers/elastic-search.ts create mode 100644 server/helpers/meilisearch.ts delete mode 100644 server/lib/elastic-search/elastic-search-channels.ts delete mode 100644 server/lib/elastic-search/elastic-search-index.ts delete mode 100644 server/lib/elastic-search/elastic-search-playlists.ts delete mode 100644 server/lib/elastic-search/elastic-search-queries.ts delete mode 100644 server/lib/elastic-search/elastic-search-videos.ts delete mode 100644 server/lib/elastic-search/shared/elastic-search-actor.ts delete mode 100644 server/lib/elastic-search/shared/elastic-search-avatar.ts delete mode 100644 server/lib/elastic-search/shared/index.ts delete mode 100644 server/lib/elastic-search/shared/query-helpers.ts create mode 100644 server/lib/meilisearch/meilisearch-channels.ts rename server/lib/{elastic-search/elastic-search-instances.ts => meilisearch/meilisearch-instances.ts} (58%) create mode 100644 server/lib/meilisearch/meilisearch-playlists.ts create mode 100644 server/lib/meilisearch/meilisearch-queries.ts create mode 100644 server/lib/meilisearch/meilisearch-videos.ts create mode 100644 server/lib/meilisearch/shared/index.ts create mode 100644 server/lib/meilisearch/shared/meilisearch-actor.ts create mode 100644 server/lib/meilisearch/shared/meilisearch-avatar.ts create mode 100644 server/lib/meilisearch/shared/query-helpers.ts create mode 100644 server/types/account.model.ts diff --git a/README.md b/README.md index 9264cbe..0110551 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ $ git submodule update --init --recursive $ yarn install --pure-lockfile ``` -The database (Elastic Search) is automatically created by PeerTube at startup. +Indexes in Meilisearch are automatically created by PeerTube at startup. Run simultaneously (for example with 3 terminals): @@ -33,7 +33,7 @@ Add the locale in `client/src/main.ts` and `client/gettext.config.js` and run `n Install dependencies: * NodeJS (v16) - * Elastic Search + * MeiliSearch ```terminal $ git clone https://framagit.org/framasoft/peertube/search-index.git /var/www/peertube-search-index @@ -44,19 +44,3 @@ $ cp config/default.yaml config/production.yaml $ vim config/production.yaml $ NODE_ENV=production NODE_CONFIG_DIR=/var/www/peertube-search-index/config node dist/server.js ``` - -### Mapping migration - -To update Elastic Search index mappings without downtime, run another instance of the search indexer -using the same configuration that the main node. You just have to update `elastic-search.indexes.*` to use new index names. - -``` -$ cd /var/www/peertube-search-index -$ cp config/production.yaml config/production-1.yaml -$ vim config/production-1.yaml -$ NODE_ENV=production NODE_APP_INSTANCE=1 NODE_CONFIG_DIR=/var/www/peertube-search-index/config node dist/server.js -``` - -After a while the new indexes will be filled. You can then stop the second indexer, update `config/production.yaml` to use -the new index names and restart the main index. - diff --git a/client/src/components/ChannelResult.vue b/client/src/components/ChannelResult.vue index 2ddebd6..9c439ca 100644 --- a/client/src/components/ChannelResult.vue +++ b/client/src/components/ChannelResult.vue @@ -105,7 +105,7 @@ return this.channel.avatar.url } - if (this.channel.avatar.length === 0) return '' + if (this.channel.avatars.length === 0) return '' const biggestAvatar = [ ...this.channel.avatars ].sort((a1, a2) => { if (a1.width < a2.width) return 1 diff --git a/config/default.yaml b/config/default.yaml index 8e857e7..137a2f8 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -7,17 +7,11 @@ webserver: hostname: 'localhost' port: 3234 -elastic-search: - # https or http - http: 'http' - auth: - username: null - password: null - ssl: - # Specificy a custom CA - ca: null - hostname: 'localhost' - port: 9200 +meilisearch: + host: 'http://127.0.0.1:7700' + + api_key: null + indexes: videos: 'peertube-index-videos' channels: 'peertube-index-channels' @@ -54,88 +48,6 @@ instances-index: enabled: false hosts: null -videos-search: - # Allow client to send browser language to boost results score that are in these languages - boost-languages: - enabled: true - - # Add ability to change videos search fields boost and match value - # See https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-multi-match-query.html for more information - # - # If boost == 0, the field will not be part of the search - # - # match_type could be 'default' or 'phrase' - # * default: use default Elastic Search match query, including fuzziness - # * phrase: use Elastic Search phrase match query - search-fields: - uuid: - boost: 100 - match_type: 'default' - short-uuid: - boost: 100 - match_type: 'default' - name: - boost: 5 - match_type: 'default' - description: - boost: 1 - match_type: 'phrase' - tags: - boost: 2 - match_type: 'default' - account-display-name: - boost: 2 - match_type: 'default' - channel-display-name: - boost: 2 - match_type: 'default' - -channels-search: - # Add ability to change channels search fields boost and match value - # See https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-multi-match-query.html for more information - # - # If boost == 0, the field will not be part of the search - # - # match_type could be 'default' or 'phrase' - # * default: use default Elastic Search match query, including fuzziness - # * phrase: use Elastic Search phrase match query - search-fields: - name: - boost: 5 - match_type: 'default' - description: - boost: 1 - match_type: 'phrase' - display-name: - boost: 3 - match_type: 'default' - account-display-name: - boost: 2 - match_type: 'default' - -playlists-search: - # Add ability to change playlists search fields boost and match value - # See https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-multi-match-query.html for more information - # - # If boost == 0, the field will not be part of the search - # - # match_type could be 'default' or 'phrase' - # * default: use default Elastic Search match query, including fuzziness - # * phrase: use Elastic Search phrase match query - search-fields: - uuid: - boost: 100 - match_type: 'default' - short-uuid: - boost: 100 - match_type: 'default' - display-name: - boost: 5 - match_type: 'default' - description: - boost: 1 - match_type: 'phrase' - api: # Blacklist hosts that will not be returned by the search API blacklist: diff --git a/config/test.yaml b/config/test.yaml index fe900ac..4e5f983 100644 --- a/config/test.yaml +++ b/config/test.yaml @@ -1,4 +1,7 @@ -elastic-search: +meilisearch: + # You can put this key as master key when starting meilisearch + api_key: '4kMeZtP0QsgE3QCDSEMYUt_WFusGjq5JgOc9atujpKw' + indexes: videos: 'peertube-index-videos-test1' channels: 'peertube-index-channels-test1' @@ -25,9 +28,7 @@ instances-index: - 'thinkerview.video' - 'replay.jres.org' - 'tube.nah.re' - - 'peertube.parleur.net' - 'video.passageenseine.fr' - - 'exode.me' api: blacklist: diff --git a/package.json b/package.json index 426f09f..730a044 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,6 @@ "i18n:update": "cd client && git fetch weblate && git merge weblate/master && npm run gettext:extract && npm run gettext:compile" }, "dependencies": { - "@elastic/elasticsearch": "^8.2.1", "async": "^3.2.3", "bluebird": "^3.5.3", "body-parser": "^1.20.0", @@ -33,6 +32,7 @@ "fs-extra": "^11.1.0", "js-yaml": "^4.1.0", "lodash": "^4.17.15", + "meilisearch": "^0.35.0", "mkdirp": "^1.0.4", "morgan": "^1.9.1", "multer": "^1.4.5-lts.1", @@ -59,7 +59,6 @@ "@types/node": "^18.11.15", "@types/pino": "^7.0.5", "@types/request": "^2.48.8", - "@types/sequelize": "^4.28.13", "@types/validator": "^13.7.2", "@typescript-eslint/eslint-plugin": "^5.27.0", "@typescript-eslint/parser": "^5.27.0", diff --git a/server/controllers/api/search-channels.ts b/server/controllers/api/search-channels.ts index 3e328ba..4240c1a 100644 --- a/server/controllers/api/search-channels.ts +++ b/server/controllers/api/search-channels.ts @@ -1,6 +1,6 @@ import express from 'express' import { Searcher } from '../../lib/controllers/searcher' -import { formatChannelForAPI, queryChannels } from '../../lib/elastic-search/elastic-search-channels' +import { formatChannelForAPI, queryChannels } from '../../lib/meilisearch/meilisearch-channels' import { asyncMiddleware } from '../../middlewares/async' import { setDefaultPagination } from '../../middlewares/pagination' import { setDefaultSearchSort } from '../../middlewares/sort' diff --git a/server/controllers/api/search-playlists.ts b/server/controllers/api/search-playlists.ts index 03cde85..5f8f1fd 100644 --- a/server/controllers/api/search-playlists.ts +++ b/server/controllers/api/search-playlists.ts @@ -1,6 +1,6 @@ import express from 'express' import { Searcher } from '../../lib/controllers/searcher' -import { formatPlaylistForAPI, queryPlaylists } from '../../lib/elastic-search/elastic-search-playlists' +import { formatPlaylistForAPI, queryPlaylists } from '../../lib/meilisearch/meilisearch-playlists' import { asyncMiddleware } from '../../middlewares/async' import { setDefaultPagination } from '../../middlewares/pagination' import { setDefaultSearchSort } from '../../middlewares/sort' diff --git a/server/controllers/api/search-videos.ts b/server/controllers/api/search-videos.ts index ff66dd8..efd6584 100644 --- a/server/controllers/api/search-videos.ts +++ b/server/controllers/api/search-videos.ts @@ -1,6 +1,6 @@ import express from 'express' import { Searcher } from '../../lib/controllers/searcher' -import { formatVideoForAPI, queryVideos } from '../../lib/elastic-search/elastic-search-videos' +import { formatVideoForAPI, queryVideos } from '../../lib/meilisearch/meilisearch-videos' import { asyncMiddleware } from '../../middlewares/async' import { setDefaultPagination } from '../../middlewares/pagination' import { setDefaultSearchSort } from '../../middlewares/sort' diff --git a/server/helpers/elastic-search.ts b/server/helpers/elastic-search.ts deleted file mode 100644 index e2b91cf..0000000 --- a/server/helpers/elastic-search.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { readFileSync } from 'fs-extra' -import { Client } from '@elastic/elasticsearch' -import { CONFIG } from '../initializers/constants' - -const elasticOptions = { - node: CONFIG.ELASTIC_SEARCH.HTTP + '://' + CONFIG.ELASTIC_SEARCH.HOSTNAME + ':' + CONFIG.ELASTIC_SEARCH.PORT -} - -if (CONFIG.ELASTIC_SEARCH.SSL.CA) { - Object.assign(elasticOptions, { - tls: { - ca: readFileSync(CONFIG.ELASTIC_SEARCH.SSL.CA) - } - }) -} - -if (CONFIG.ELASTIC_SEARCH.AUTH.USERNAME) { - Object.assign(elasticOptions, { - auth: { - username: CONFIG.ELASTIC_SEARCH.AUTH.USERNAME, - password: CONFIG.ELASTIC_SEARCH.AUTH.PASSWORD - } - }) -} - -const elasticSearch = new Client(elasticOptions) - -export { - elasticSearch -} diff --git a/server/helpers/meilisearch.ts b/server/helpers/meilisearch.ts new file mode 100644 index 0000000..550281a --- /dev/null +++ b/server/helpers/meilisearch.ts @@ -0,0 +1,9 @@ +import { MeiliSearch } from 'meilisearch' +import { CONFIG } from '../initializers/constants' + +const client = new MeiliSearch({ + host: CONFIG.MEILISEARCH.HOST, + apiKey: CONFIG.MEILISEARCH.API_KEY +}) + +export { client } diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index e181db1..d69275c 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -12,21 +12,13 @@ const CONFIG = { HOSTNAME: config.get('webserver.hostname'), PORT: config.get('webserver.port') }, - ELASTIC_SEARCH: { - HTTP: config.get('elastic-search.http'), - AUTH: { - USERNAME: config.get('elastic-search.auth.username'), - PASSWORD: config.get('elastic-search.auth.password') - }, - SSL: { - CA: config.get('elastic-search.ssl.ca') - }, - HOSTNAME: config.get('elastic-search.hostname'), - PORT: config.get('elastic-search.port'), + MEILISEARCH: { + HOST: config.get('meilisearch.host'), + API_KEY: config.get('meilisearch.api_key'), INDEXES: { - VIDEOS: config.get('elastic-search.indexes.videos'), - CHANNELS: config.get('elastic-search.indexes.channels'), - PLAYLISTS: config.get('elastic-search.indexes.playlists') + VIDEOS: config.get('meilisearch.indexes.videos'), + CHANNELS: config.get('meilisearch.indexes.channels'), + PLAYLISTS: config.get('meilisearch.indexes.playlists') } }, LOG: { @@ -40,96 +32,6 @@ const CONFIG = { LEGAL_NOTICES_URL: config.get('search-instance.legal_notices_url'), THEME: config.get('search-instance.theme') }, - VIDEOS_SEARCH: { - BOOST_LANGUAGES: { - ENABLED: config.get('videos-search.boost-languages.enabled') - }, - SEARCH_FIELDS: { - UUID: { - FIELD_NAME: 'uuid', - BOOST: config.get('videos-search.search-fields.uuid.boost'), - MATCH_TYPE: config.get('videos-search.search-fields.uuid.match_type') - }, - SHORT_UUID: { - FIELD_NAME: 'shortUUID', - BOOST: config.get('videos-search.search-fields.short-uuid.boost'), - MATCH_TYPE: config.get('videos-search.search-fields.short-uuid.match_type') - }, - NAME: { - FIELD_NAME: 'name', - BOOST: config.get('videos-search.search-fields.name.boost'), - MATCH_TYPE: config.get('videos-search.search-fields.name.match_type') - }, - DESCRIPTION: { - FIELD_NAME: 'description', - BOOST: config.get('videos-search.search-fields.description.boost'), - MATCH_TYPE: config.get('videos-search.search-fields.description.match_type') - }, - TAGS: { - FIELD_NAME: 'tags', - BOOST: config.get('videos-search.search-fields.tags.boost'), - MATCH_TYPE: config.get('videos-search.search-fields.tags.match_type') - }, - ACCOUNT_DISPLAY_NAME: { - FIELD_NAME: 'account.displayName', - BOOST: config.get('videos-search.search-fields.account-display-name.boost'), - MATCH_TYPE: config.get('videos-search.search-fields.account-display-name.match_type') - }, - CHANNEL_DISPLAY_NAME: { - FIELD_NAME: 'channel.displayName', - BOOST: config.get('videos-search.search-fields.channel-display-name.boost'), - MATCH_TYPE: config.get('videos-search.search-fields.channel-display-name.match_type') - } - } - }, - CHANNELS_SEARCH: { - SEARCH_FIELDS: { - NAME: { - FIELD_NAME: 'name', - BOOST: config.get('channels-search.search-fields.name.boost'), - MATCH_TYPE: config.get('channels-search.search-fields.name.match_type') - }, - DESCRIPTION: { - FIELD_NAME: 'description', - BOOST: config.get('channels-search.search-fields.description.boost'), - MATCH_TYPE: config.get('channels-search.search-fields.description.match_type') - }, - DISPLAY_NAME: { - FIELD_NAME: 'displayName', - BOOST: config.get('channels-search.search-fields.display-name.boost'), - MATCH_TYPE: config.get('channels-search.search-fields.display-name.match_type') - }, - ACCOUNT_DISPLAY_NAME: { - FIELD_NAME: 'ownerAccount.displayName', - BOOST: config.get('channels-search.search-fields.account-display-name.boost'), - MATCH_TYPE: config.get('channels-search.search-fields.account-display-name.match_type') - } - } - }, - PLAYLISTS_SEARCH: { - SEARCH_FIELDS: { - UUID: { - FIELD_NAME: 'uuid', - BOOST: config.get('playlists-search.search-fields.uuid.boost'), - MATCH_TYPE: config.get('playlists-search.search-fields.uuid.match_type') - }, - SHORT_UUID: { - FIELD_NAME: 'shortUUID', - BOOST: config.get('playlists-search.search-fields.short-uuid.boost'), - MATCH_TYPE: config.get('playlists-search.search-fields.short-uuid.match_type') - }, - DISPLAY_NAME: { - FIELD_NAME: 'displayName', - BOOST: config.get('playlists-search.search-fields.display-name.boost'), - MATCH_TYPE: config.get('playlists-search.search-fields.display-name.match_type') - }, - DESCRIPTION: { - FIELD_NAME: 'description', - BOOST: config.get('playlists-search.search-fields.description.boost'), - MATCH_TYPE: config.get('playlists-search.search-fields.description.match_type') - } - } - }, INSTANCES_INDEX: { URL: config.get('instances-index.url'), PUBLIC_URL: config.get('instances-index.public_url'), @@ -147,9 +49,9 @@ const CONFIG = { } const SORTABLE_COLUMNS = { - VIDEOS_SEARCH: [ 'name', 'duration', 'createdAt', 'publishedAt', 'originallyPublishedAt', 'views', 'likes', 'match' ], - CHANNELS_SEARCH: [ 'match', 'displayName', 'createdAt' ], - PLAYLISTS_SEARCH: [ 'match', 'displayName', 'createdAt' ] + VIDEOS_SEARCH: [ '_rankingScore', 'match', 'name', 'duration', 'createdAt', 'publishedAt', 'originallyPublishedAt', 'views', 'likes' ], + CHANNELS_SEARCH: [ '_rankingScore', 'match', 'displayName', 'createdAt' ], + PLAYLISTS_SEARCH: [ '_rankingScore', 'match', 'displayName', 'createdAt' ] } const PAGINATION_START = { @@ -165,7 +67,7 @@ const SCHEDULER_INTERVALS_MS = { indexation: 60000 * 60 * 24 // 24 hours } -const INDEXER_COUNT = 10 +const INDEXER_COUNT = 20 const INDEXER_LIMIT = 500000 const INDEXER_HOST_CONCURRENCY = 3 @@ -176,17 +78,6 @@ const REQUESTS = { WAIT: 10000 // 10 seconds } -const ELASTIC_SEARCH_QUERY = { - FUZZINESS: 'AUTO:4,7', - OPERATOR: 'OR', - MINIMUM_SHOULD_MATCH: '3<75%', - BOOST_LANGUAGE_VALUE: 1, - MALUS_LANGUAGE_VALUE: 0.5, - VIDEOS_MULTI_MATCH_FIELDS: buildMatchFieldConfig(CONFIG.VIDEOS_SEARCH.SEARCH_FIELDS), - CHANNELS_MULTI_MATCH_FIELDS: buildMatchFieldConfig(CONFIG.CHANNELS_SEARCH.SEARCH_FIELDS), - PLAYLISTS_MULTI_MATCH_FIELDS: buildMatchFieldConfig(CONFIG.PLAYLISTS_SEARCH.SEARCH_FIELDS) -} - function getWebserverUrl () { if (CONFIG.WEBSERVER.PORT === 80 || CONFIG.WEBSERVER.PORT === 443) { return CONFIG.WEBSERVER.SCHEME + '://' + CONFIG.WEBSERVER.HOSTNAME @@ -195,28 +86,6 @@ function getWebserverUrl () { return CONFIG.WEBSERVER.SCHEME + '://' + CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT } -function buildMatchFieldConfig (fields: { [name: string]: { BOOST: number, FIELD_NAME: string, MATCH_TYPE: string } }) { - const selectFields = (matchType: 'phrase' | 'default') => { - return Object.keys(fields) - .filter(fieldName => fields[fieldName].MATCH_TYPE === matchType) - .map(fieldName => fields[fieldName]) - } - - const buildMultiMatch = (fields: { BOOST: number, FIELD_NAME: string }[]) => { - return fields.map(fieldObj => { - if (fieldObj.BOOST <= 0) return '' - - return `${fieldObj.FIELD_NAME}^${fieldObj.BOOST}` - }) - .filter(v => !!v) - } - - return { - default: buildMultiMatch(selectFields('default')), - phrase: buildMultiMatch(selectFields('phrase')) - } -} - if (isTestInstance()) { SCHEDULER_INTERVALS_MS.indexation = 1000 * 60 * 5 // 5 minutes } @@ -234,6 +103,5 @@ export { INDEXER_HOST_CONCURRENCY, INDEXER_COUNT, INDEXER_LIMIT, - REQUESTS, - ELASTIC_SEARCH_QUERY + REQUESTS } diff --git a/server/lib/elastic-search/elastic-search-channels.ts b/server/lib/elastic-search/elastic-search-channels.ts deleted file mode 100644 index feeba1e..0000000 --- a/server/lib/elastic-search/elastic-search-channels.ts +++ /dev/null @@ -1,209 +0,0 @@ -import { MappingProperty, PropertyName } from '@elastic/elasticsearch/lib/api/types' -import { elasticSearch } from '../../helpers/elastic-search' -import { logger } from '../../helpers/logger' -import { CONFIG, ELASTIC_SEARCH_QUERY } from '../../initializers/constants' -import { DBChannel, EnhancedVideoChannel, IndexableChannel } from '../../types/channel.model' -import { ChannelsSearchQuery } from '../../types/search-query/channel-search.model' -import { buildSort, extractSearchQueryResult } from './elastic-search-queries' -import { buildChannelOrAccountCommonMapping, buildMultiMatchBool } from './shared' -import { - formatActorImageForAPI, - formatActorImageForDB, - formatActorImagesForAPI, - formatActorImagesForDB -} from './shared/elastic-search-avatar' - -async function queryChannels (search: ChannelsSearchQuery) { - const bool: any = {} - const mustNot: any[] = [] - const filter: any[] = [] - - if (search.search) { - Object.assign(bool, buildMultiMatchBool(search.search, ELASTIC_SEARCH_QUERY.CHANNELS_MULTI_MATCH_FIELDS)) - } - - if (search.blockedAccounts) { - mustNot.push({ - terms: { - 'ownerAccount.handle': search.blockedAccounts - } - }) - } - - if (search.blockedHosts) { - mustNot.push({ - terms: { - host: search.blockedHosts - } - }) - } - - mustNot.push({ - term: { - videosCount: 0 - } - }) - - if (search.host) { - filter.push({ - term: { - host: search.host - } - }) - } - - if (search.handles) { - filter.push({ - terms: { - handle: search.handles - } - }) - } - - if (filter.length !== 0) { - Object.assign(bool, { filter }) - } - - if (mustNot.length !== 0) { - Object.assign(bool, { must_not: mustNot }) - } - - const body = { - from: search.start, - size: search.count, - sort: buildSort(search.sort), - query: { bool } - } - - logger.debug({ body }, 'Will query Elastic Search for channels.') - - const res = await elasticSearch.search({ - index: CONFIG.ELASTIC_SEARCH.INDEXES.CHANNELS, - body - }) - - return extractSearchQueryResult(res) -} - -function formatChannelForAPI (c: DBChannel, fromHost?: string): EnhancedVideoChannel { - return { - id: c.id, - - score: c.score, - - url: c.url, - name: c.name, - host: c.host, - followingCount: c.followingCount, - followersCount: c.followersCount, - createdAt: c.createdAt, - updatedAt: c.updatedAt, - - avatar: formatActorImageForAPI(c.avatar), - avatars: formatActorImagesForAPI(c.avatars, c.avatar), - - banner: formatActorImageForAPI(c.banner), - banners: formatActorImagesForAPI(c.banners, c.banner), - - displayName: c.displayName, - description: c.description, - support: c.support, - isLocal: fromHost === c.host, - - videosCount: c.videosCount || 0, - - ownerAccount: { - id: c.ownerAccount.id, - url: c.ownerAccount.url, - - displayName: c.ownerAccount.displayName, - description: c.ownerAccount.description, - name: c.ownerAccount.name, - host: c.ownerAccount.host, - followingCount: c.ownerAccount.followingCount, - followersCount: c.ownerAccount.followersCount, - createdAt: c.ownerAccount.createdAt, - updatedAt: c.ownerAccount.updatedAt, - - avatar: formatActorImageForAPI(c.ownerAccount.avatar), - avatars: formatActorImagesForAPI(c.ownerAccount.avatars, c.ownerAccount.avatar) - } - } -} - -function formatChannelForDB (c: IndexableChannel): DBChannel { - return { - id: c.id, - - name: c.name, - host: c.host, - url: c.url, - - avatar: formatActorImageForDB(c.avatar, c.host), - avatars: formatActorImagesForDB(c.avatars, c.host), - - banner: formatActorImageForDB(c.banner, c.host), - banners: formatActorImagesForDB(c.banners, c.host), - - displayName: c.displayName, - - indexedAt: new Date(), - - followingCount: c.followingCount, - followersCount: c.followersCount, - createdAt: c.createdAt, - updatedAt: c.updatedAt, - - description: c.description, - support: c.support, - videosCount: c.videosCount, - - handle: `${c.name}@${c.host}`, - - ownerAccount: { - id: c.ownerAccount.id, - url: c.ownerAccount.url, - - displayName: c.ownerAccount.displayName, - description: c.ownerAccount.description, - name: c.ownerAccount.name, - host: c.ownerAccount.host, - followingCount: c.ownerAccount.followingCount, - followersCount: c.ownerAccount.followersCount, - createdAt: c.ownerAccount.createdAt, - updatedAt: c.ownerAccount.updatedAt, - - handle: `${c.ownerAccount.name}@${c.ownerAccount.host}`, - - avatar: formatActorImageForDB(c.ownerAccount.avatar, c.ownerAccount.host), - avatars: formatActorImagesForDB(c.ownerAccount.avatars, c.ownerAccount.host) - } - } -} - -function buildChannelsMapping () { - const base = buildChannelOrAccountCommonMapping() - - Object.assign(base, { - videosCount: { - type: 'long' - }, - - support: { - type: 'keyword' - }, - - ownerAccount: { - properties: buildChannelOrAccountCommonMapping() - } - } as Record) - - return base -} - -export { - buildChannelsMapping, - formatChannelForDB, - formatChannelForAPI, - queryChannels -} diff --git a/server/lib/elastic-search/elastic-search-index.ts b/server/lib/elastic-search/elastic-search-index.ts deleted file mode 100644 index cdc2aae..0000000 --- a/server/lib/elastic-search/elastic-search-index.ts +++ /dev/null @@ -1,91 +0,0 @@ -import { flatMap } from 'lodash' -import { MappingProperty, PropertyName } from '@elastic/elasticsearch/lib/api/types' -import { elasticSearch } from '../../helpers/elastic-search' -import { logger } from '../../helpers/logger' -import { IndexableDoc } from '../../types/indexable-doc.model' - -function buildIndex (name: string, mapping: Record) { - logger.info('Initialize %s Elastic Search index.', name) - - return elasticSearch.indices.create({ - index: name, - body: { - settings: { - number_of_shards: 1, - number_of_replicas: 1 - }, - mappings: { - properties: mapping - } - } - }).catch(err => { - if (err.name === 'ResponseError' && err.meta?.body?.error.root_cause[0]?.type === 'resource_already_exists_exception') return - - throw err - }) -} - -async function indexDocuments (options: { - objects: T[] - formatter: (o: T) => any - replace: boolean - index: string -}) { - const { objects, formatter, replace, index } = options - - const elIdIndex: { [elId: string]: T } = {} - - for (const object of objects) { - elIdIndex[object.elasticSearchId] = object - } - - const method = replace ? 'index' : 'update' - - const body = flatMap(objects, v => { - const doc = formatter(v) - - const options = replace - ? doc - : { doc, doc_as_upsert: true } - - return [ - { - [method]: { - _id: v.elasticSearchId, - _index: index - } - }, - options - ] - }) - - const result = await elasticSearch.bulk({ - index, - body - }) - - if (result.errors === true) { - const msg = 'Cannot insert data in elastic search.' - logger.error({ err: result }, msg) - throw new Error(msg) - } - - const created: T[] = result.items - .map(i => i[method]) - .filter(i => i.result === 'created') - .map(i => elIdIndex[i._id]) - - return { created } -} - -function refreshIndex (indexName: string) { - logger.info('Refreshing %s index.', indexName) - - return elasticSearch.indices.refresh({ index: indexName }) -} - -export { - buildIndex, - indexDocuments, - refreshIndex -} diff --git a/server/lib/elastic-search/elastic-search-playlists.ts b/server/lib/elastic-search/elastic-search-playlists.ts deleted file mode 100644 index 4c450f1..0000000 --- a/server/lib/elastic-search/elastic-search-playlists.ts +++ /dev/null @@ -1,234 +0,0 @@ -import { MappingProperty, PropertyName } from '@elastic/elasticsearch/lib/api/types' -import { elasticSearch } from '../../helpers/elastic-search' -import { logger } from '../../helpers/logger' -import { buildUrl } from '../../helpers/utils' -import { CONFIG, ELASTIC_SEARCH_QUERY } from '../../initializers/constants' -import { DBPlaylist, EnhancedPlaylist, IndexablePlaylist } from '../../types/playlist.model' -import { PlaylistsSearchQuery } from '../../types/search-query/playlist-search.model' -import { buildSort, extractSearchQueryResult } from './elastic-search-queries' -import { addUUIDFilters, buildMultiMatchBool } from './shared' -import { buildChannelOrAccountSummaryMapping, formatActorForDB, formatActorSummaryForAPI } from './shared/elastic-search-actor' - -async function queryPlaylists (search: PlaylistsSearchQuery) { - const bool: any = {} - const mustNot: any[] = [] - const filter: any[] = [] - - if (search.search) { - Object.assign(bool, buildMultiMatchBool(search.search, ELASTIC_SEARCH_QUERY.PLAYLISTS_MULTI_MATCH_FIELDS)) - } - - if (search.blockedAccounts) { - mustNot.push({ - terms: { - 'ownerAccount.handle': search.blockedAccounts - } - }) - } - - if (search.blockedHosts) { - mustNot.push({ - terms: { - host: search.blockedHosts - } - }) - } - - mustNot.push({ - term: { - videosLength: 0 - } - }) - - if (search.host) { - filter.push({ - term: { - 'ownerAccount.host': search.host - } - }) - } - - if (search.uuids) { - addUUIDFilters(filter, search.uuids) - } - - if (filter.length !== 0) { - Object.assign(bool, { filter }) - } - - if (mustNot.length !== 0) { - Object.assign(bool, { must_not: mustNot }) - } - - const body = { - from: search.start, - size: search.count, - sort: buildSort(search.sort), - query: { bool } - } - - logger.debug({ body }, 'Will query Elastic Search for playlists.') - - const res = await elasticSearch.search({ - index: CONFIG.ELASTIC_SEARCH.INDEXES.PLAYLISTS, - body - }) - - return extractSearchQueryResult(res) -} - -function formatPlaylistForAPI (p: DBPlaylist, fromHost?: string): EnhancedPlaylist { - return { - id: p.id, - uuid: p.uuid, - shortUUID: p.shortUUID, - - score: p.score, - - isLocal: fromHost === p.host, - - url: p.url, - - displayName: p.displayName, - description: p.description, - - privacy: { - id: p.privacy.id, - label: p.privacy.label - }, - - videosLength: p.videosLength, - - type: { - id: p.type.id, - label: p.type.label - }, - - thumbnailPath: p.thumbnailPath, - thumbnailUrl: buildUrl(p.host, p.thumbnailPath), - - embedPath: p.embedPath, - embedUrl: buildUrl(p.host, p.embedPath), - - createdAt: p.createdAt, - updatedAt: p.updatedAt, - - ownerAccount: formatActorSummaryForAPI(p.ownerAccount), - videoChannel: formatActorSummaryForAPI(p.videoChannel) - } -} - -function formatPlaylistForDB (p: IndexablePlaylist): DBPlaylist { - return { - id: p.id, - uuid: p.uuid, - shortUUID: p.shortUUID, - - indexedAt: new Date(), - createdAt: p.createdAt, - updatedAt: p.updatedAt, - - host: p.host, - url: p.url, - - displayName: p.displayName, - description: p.description, - - thumbnailPath: p.thumbnailPath, - embedPath: p.embedPath, - - type: { - id: p.type.id, - label: p.type.label - }, - privacy: { - id: p.privacy.id, - label: p.privacy.label - }, - - videosLength: p.videosLength, - - ownerAccount: formatActorForDB(p.ownerAccount), - videoChannel: formatActorForDB(p.videoChannel) - } -} - -function buildPlaylistsMapping () { - return { - id: { - type: 'long' - }, - - uuid: { - type: 'keyword' - }, - shortUUID: { - type: 'keyword' - }, - createdAt: { - type: 'date', - format: 'date_optional_time' - }, - updatedAt: { - type: 'date', - format: 'date_optional_time' - }, - indexedAt: { - type: 'date', - format: 'date_optional_time' - }, - - privacy: { - properties: { - id: { - type: 'keyword' - }, - label: { - type: 'text' - } - } - }, - - displayName: { - type: 'text' - }, - - description: { - type: 'text' - }, - - thumbnailPath: { - type: 'keyword' - }, - embedPath: { - type: 'keyword' - }, - - url: { - type: 'keyword' - }, - - host: { - type: 'keyword' - }, - - videosLength: { - type: 'long' - }, - - ownerAccount: { - properties: buildChannelOrAccountSummaryMapping() - }, - - videoChannel: { - properties: buildChannelOrAccountSummaryMapping() - } - } as Record -} - -export { - formatPlaylistForAPI, - buildPlaylistsMapping, - formatPlaylistForDB, - queryPlaylists -} diff --git a/server/lib/elastic-search/elastic-search-queries.ts b/server/lib/elastic-search/elastic-search-queries.ts deleted file mode 100644 index bda2e5e..0000000 --- a/server/lib/elastic-search/elastic-search-queries.ts +++ /dev/null @@ -1,134 +0,0 @@ -import { difference } from 'lodash' -import { estypes } from '@elastic/elasticsearch' -import { AggregationsBuckets, AggregationsStringTermsAggregate, AggregationsStringTermsBucket } from '@elastic/elasticsearch/lib/api/types' -import { elasticSearch } from '../../helpers/elastic-search' -import { logger } from '../../helpers/logger' - -async function removeNotExistingIdsFromHost (indexName: string, host: string, existingIds: Set) { - const idsFromDB = await getIdsOf(indexName, host) - - // Elastic Search limits max terms amount - const idsToRemove = difference(idsFromDB, Array.from(existingIds)).splice(0, 50000) - - logger.info({ idsToRemove }, 'Will remove %d entries from %s of host %s.', idsToRemove.length, indexName, host) - - return elasticSearch.deleteByQuery({ - index: indexName, - body: { - query: { - bool: { - filter: [ - { - terms: { - id: idsToRemove - } - }, - { - term: { - host - } - } - ] - } - } - } - }) -} - -function removeFromHosts (indexName: string, hosts: string[]) { - if (hosts.length === 0) return - - logger.info({ hosts }, 'Will remove entries of index %s from hosts.', indexName) - - return elasticSearch.deleteByQuery({ - index: indexName, - body: { - query: { - bool: { - filter: { - terms: { - host: hosts - } - } - } - } - } - }) -} - -async function getIdsOf (indexName: string, host: string) { - const res = await elasticSearch.search>({ - index: indexName, - body: { - size: 0, - aggs: { - ids: { - terms: { - size: 500000, - field: 'id' - } - } - }, - query: { - bool: { - filter: [ - { - term: { - host - } - } - ] - } - } - } - }) - - return extractBucketsFromAggregation(res.aggregations.ids.buckets).map(b => b.key) -} - -function extractSearchQueryResult (result: estypes.SearchResponse) { - const hits = result.hits - - return { - total: (hits.total as estypes.SearchTotalHits).value, - data: hits.hits.map(h => Object.assign(h._source, { score: h._score })) - } -} - -function extractBucketsFromAggregation (buckets: AggregationsBuckets) { - // FIXME: key returned by elastic search can also be a number - return buckets as unknown as { key: T }[] -} - -function buildSort (value: string) { - let sortField: string - let direction: 'asc' | 'desc' - - if (value.substring(0, 1) === '-') { - direction = 'desc' - sortField = value.substring(1) - } else { - direction = 'asc' - sortField = value - } - - const field = sortField === 'match' - ? '_score' - : sortField - - return [ - { - [field]: { order: direction } - } - ] -} - -export { - elasticSearch, - removeNotExistingIdsFromHost, - getIdsOf, - extractSearchQueryResult, - removeFromHosts, - buildSort, - extractBucketsFromAggregation -} diff --git a/server/lib/elastic-search/elastic-search-videos.ts b/server/lib/elastic-search/elastic-search-videos.ts deleted file mode 100644 index 03247f0..0000000 --- a/server/lib/elastic-search/elastic-search-videos.ts +++ /dev/null @@ -1,528 +0,0 @@ -import { MappingProperty, PropertyName } from '@elastic/elasticsearch/lib/api/types' -import { exists } from '../../helpers/custom-validators/misc' -import { elasticSearch } from '../../helpers/elastic-search' -import { logger } from '../../helpers/logger' -import { buildUrl } from '../../helpers/utils' -import { CONFIG, ELASTIC_SEARCH_QUERY } from '../../initializers/constants' -import { VideosSearchQuery } from '../../types/search-query/video-search.model' -import { DBVideo, DBVideoDetails, EnhancedVideo, IndexableVideo, IndexableVideoDetails } from '../../types/video.model' -import { buildSort, extractSearchQueryResult } from './elastic-search-queries' -import { addUUIDFilters, buildMultiMatchBool } from './shared' -import { buildChannelOrAccountSummaryMapping, formatActorForDB, formatActorSummaryForAPI } from './shared/elastic-search-actor' - -async function queryVideos (search: VideosSearchQuery) { - const bool: any = {} - const filter: any[] = [] - const mustNot: any[] = [] - - if (search.search) { - Object.assign(bool, buildMultiMatchBool(search.search, ELASTIC_SEARCH_QUERY.VIDEOS_MULTI_MATCH_FIELDS)) - } - - if (search.blockedAccounts) { - mustNot.push({ - terms: { - 'account.handle': search.blockedAccounts - } - }) - } - - if (search.blockedHosts) { - mustNot.push({ - terms: { - host: search.blockedHosts - } - }) - } - - if (search.startDate) { - filter.push({ - range: { - publishedAt: { - gte: search.startDate - } - } - }) - } - - if (search.endDate) { - filter.push({ - range: { - publishedAt: { - lte: search.endDate - } - } - }) - } - - if (search.originallyPublishedStartDate) { - filter.push({ - range: { - originallyPublishedAt: { - gte: search.startDate - } - } - }) - } - - if (search.originallyPublishedEndDate) { - filter.push({ - range: { - originallyPublishedAt: { - lte: search.endDate - } - } - }) - } - - if (search.nsfw && search.nsfw !== 'both') { - filter.push({ - term: { - nsfw: (search.nsfw + '') === 'true' - } - }) - } - - if (search.categoryOneOf) { - filter.push({ - terms: { - 'category.id': search.categoryOneOf - } - }) - } - - if (search.licenceOneOf) { - filter.push({ - terms: { - 'licence.id': search.licenceOneOf - } - }) - } - - if (search.languageOneOf) { - filter.push({ - terms: { - 'language.id': search.languageOneOf - } - }) - } - - if (search.tagsOneOf) { - filter.push({ - terms: { - tags: search.tagsOneOf.map(t => t.toLowerCase()) - } - }) - } - - if (search.tagsAllOf) { - for (const t of search.tagsAllOf) { - filter.push({ - term: { - 'tags.raw': { - value: t, - case_insensitive: true - } - } - }) - } - } - - if (search.durationMin) { - filter.push({ - range: { - duration: { - gte: search.durationMin - } - } - }) - } - - if (search.durationMax) { - filter.push({ - range: { - duration: { - lte: search.durationMax - } - } - }) - } - - if (exists(search.isLive)) { - filter.push({ - term: { - isLive: search.isLive - } - }) - } - - if (search.host) { - filter.push({ - term: { - 'account.host': search.host - } - }) - } - - if (search.uuids) { - addUUIDFilters(filter, search.uuids) - } - - Object.assign(bool, { filter }) - - if (mustNot.length !== 0) { - Object.assign(bool, { must_not: mustNot }) - } - - const body = { - from: search.start, - size: search.count, - sort: buildSort(search.sort) - } - - // Allow to boost results depending on query languages - if ( - CONFIG.VIDEOS_SEARCH.BOOST_LANGUAGES.ENABLED && - Array.isArray(search.boostLanguages) && - search.boostLanguages.length !== 0 - ) { - const boostScript = ` - if (doc['language.id'].size() == 0) { - return _score; - } - - String language = doc['language.id'].value; - - for (String docLang: params.boostLanguages) { - if (docLang == language) return _score * params.boost; - } - - return _score * params.malus; - ` - - Object.assign(body, { - query: { - script_score: { - query: { bool }, - script: { - source: boostScript, - params: { - boostLanguages: search.boostLanguages, - boost: ELASTIC_SEARCH_QUERY.BOOST_LANGUAGE_VALUE, - malus: ELASTIC_SEARCH_QUERY.MALUS_LANGUAGE_VALUE - } - } - } - } - }) - } else { - Object.assign(body, { query: { bool } }) - } - - logger.debug({ body }, 'Will query Elastic Search for videos.') - - const res = await elasticSearch.search({ - index: CONFIG.ELASTIC_SEARCH.INDEXES.VIDEOS, - body - }) - - return extractSearchQueryResult(res) -} - -function buildVideosMapping () { - return { - id: { - type: 'long' - }, - - uuid: { - type: 'keyword' - }, - shortUUID: { - type: 'keyword' - }, - createdAt: { - type: 'date', - format: 'date_optional_time' - }, - updatedAt: { - type: 'date', - format: 'date_optional_time' - }, - publishedAt: { - type: 'date', - format: 'date_optional_time' - }, - originallyPublishedAt: { - type: 'date', - format: 'date_optional_time' - }, - indexedAt: { - type: 'date', - format: 'date_optional_time' - }, - - category: { - properties: { - id: { - type: 'keyword' - }, - label: { - type: 'text' - } - } - }, - - licence: { - properties: { - id: { - type: 'keyword' - }, - label: { - type: 'text' - } - } - }, - - language: { - properties: { - id: { - type: 'keyword' - }, - label: { - type: 'text' - } - } - }, - - privacy: { - properties: { - id: { - type: 'keyword' - }, - label: { - type: 'text' - } - } - }, - - name: { - type: 'text' - }, - - description: { - type: 'text' - }, - - tags: { - type: 'text', - - fields: { - raw: { - type: 'keyword' - } - } - }, - - duration: { - type: 'long' - }, - - thumbnailPath: { - type: 'keyword' - }, - previewPath: { - type: 'keyword' - }, - embedPath: { - type: 'keyword' - }, - - url: { - type: 'keyword' - }, - - views: { - type: 'long' - }, - likes: { - type: 'long' - }, - dislikes: { - type: 'long' - }, - nsfw: { - type: 'boolean' - }, - isLive: { - type: 'boolean' - }, - - host: { - type: 'keyword' - }, - - account: { - properties: buildChannelOrAccountSummaryMapping() - }, - - channel: { - properties: buildChannelOrAccountSummaryMapping() - } - } as Record -} - -function formatVideoForDB (v: IndexableVideo | IndexableVideoDetails): DBVideo | DBVideoDetails { - const video = { - id: v.id, - uuid: v.uuid, - shortUUID: v.shortUUID, - - indexedAt: new Date(), - createdAt: v.createdAt, - updatedAt: v.updatedAt, - publishedAt: v.publishedAt, - originallyPublishedAt: v.originallyPublishedAt, - - category: { - id: v.category.id, - label: v.category.label - }, - licence: { - id: v.licence.id, - label: v.licence.label - }, - language: { - id: v.language.id, - label: v.language.label - }, - privacy: { - id: v.privacy.id, - label: v.privacy.label - }, - - name: v.name, - - truncatedDescription: v.truncatedDescription, - description: v.description, - - waitTranscoding: v.waitTranscoding, - - duration: v.duration, - - thumbnailPath: v.thumbnailPath, - previewPath: v.previewPath, - embedPath: v.embedPath, - - views: v.views, - viewers: v.viewers, - likes: v.likes, - dislikes: v.dislikes, - - isLive: v.isLive || false, - nsfw: v.nsfw, - - host: v.host, - url: v.url, - - files: v.files, - streamingPlaylists: v.streamingPlaylists, - - tags: (v as IndexableVideoDetails).tags ? (v as IndexableVideoDetails).tags : undefined, - - account: formatActorForDB(v.account), - channel: formatActorForDB(v.channel) - } - - if (isVideoDetails(v)) { - return { - ...video, - trackerUrls: v.trackerUrls, - - descriptionPath: v.descriptionPath, - - support: v.support, - - commentsEnabled: v.commentsEnabled, - downloadEnabled: v.downloadEnabled - } - } - - return video -} - -function formatVideoForAPI (v: DBVideoDetails, fromHost?: string): EnhancedVideo { - return { - id: v.id, - uuid: v.uuid, - shortUUID: v.shortUUID, - - score: v.score, - - createdAt: new Date(v.createdAt), - updatedAt: new Date(v.updatedAt), - publishedAt: new Date(v.publishedAt), - originallyPublishedAt: v.originallyPublishedAt, - - category: { - id: v.category.id, - label: v.category.label - }, - licence: { - id: v.licence.id, - label: v.licence.label - }, - language: { - id: v.language.id, - label: v.language.label - }, - privacy: { - id: v.privacy.id, - label: v.privacy.label - }, - - name: v.name, - description: v.description, - truncatedDescription: v.truncatedDescription, - duration: v.duration, - - tags: v.tags, - - thumbnailPath: v.thumbnailPath, - thumbnailUrl: buildUrl(v.host, v.thumbnailPath), - - previewPath: v.previewPath, - previewUrl: buildUrl(v.host, v.previewPath), - - embedPath: v.embedPath, - embedUrl: buildUrl(v.host, v.embedPath), - - url: v.url, - - isLocal: fromHost && fromHost === v.host, - - views: v.views, - viewers: v.viewers, - likes: v.likes, - dislikes: v.dislikes, - - isLive: v.isLive, - nsfw: v.nsfw, - - account: formatActorSummaryForAPI(v.account), - channel: formatActorSummaryForAPI(v.channel) - } -} - -export { - queryVideos, - formatVideoForDB, - formatVideoForAPI, - buildVideosMapping -} - -// --------------------------------------------------------------------------- - -function isVideoDetails (video: IndexableVideo | IndexableVideoDetails): video is IndexableVideoDetails { - return (video as IndexableVideoDetails).commentsEnabled !== undefined -} diff --git a/server/lib/elastic-search/shared/elastic-search-actor.ts b/server/lib/elastic-search/shared/elastic-search-actor.ts deleted file mode 100644 index 5dd65c2..0000000 --- a/server/lib/elastic-search/shared/elastic-search-actor.ts +++ /dev/null @@ -1,104 +0,0 @@ -import { MappingProperty, PropertyName } from '@elastic/elasticsearch/lib/api/types' -import { AccountSummary, VideoChannelSummary } from '@peertube/peertube-types' -import { AdditionalActorAttributes } from '../../../types/actor.model' -import { formatActorImageForDB } from './' -import { buildActorImageMapping, formatActorImageForAPI, formatActorImagesForAPI, formatActorImagesForDB } from './elastic-search-avatar' - -function buildChannelOrAccountSummaryMapping () { - return { - id: { - type: 'long' - }, - - name: { - type: 'text', - fields: { - raw: { - type: 'keyword' - } - } - }, - displayName: { - type: 'text' - }, - url: { - type: 'keyword' - }, - host: { - type: 'keyword' - }, - handle: { - type: 'keyword' - }, - - avatar: { - properties: buildActorImageMapping() - }, - - // Introduced in 4.2 - avatars: { - properties: buildActorImageMapping() - } - } as Record -} - -function buildChannelOrAccountCommonMapping () { - return { - ...buildChannelOrAccountSummaryMapping(), - - followingCount: { - type: 'long' - }, - followersCount: { - type: 'long' - }, - - createdAt: { - type: 'date', - format: 'date_optional_time' - }, - updatedAt: { - type: 'date', - format: 'date_optional_time' - }, - - description: { - type: 'text' - } - } as Record -} - -function formatActorSummaryForAPI (actor: (AccountSummary | VideoChannelSummary) & AdditionalActorAttributes) { - return { - id: actor.id, - name: actor.name, - displayName: actor.displayName, - url: actor.url, - host: actor.host, - - avatar: formatActorImageForAPI(actor.avatar), - avatars: formatActorImagesForAPI(actor.avatars, actor.avatar) - } -} - -function formatActorForDB (actor: AccountSummary | VideoChannelSummary) { - return { - id: actor.id, - name: actor.name, - displayName: actor.displayName, - url: actor.url, - host: actor.host, - - handle: `${actor.name}@${actor.host}`, - - avatar: formatActorImageForDB(actor.avatar, actor.host), - avatars: formatActorImagesForDB(actor.avatars, actor.host) - } -} - -export { - buildChannelOrAccountCommonMapping, - buildChannelOrAccountSummaryMapping, - formatActorSummaryForAPI, - formatActorForDB -} diff --git a/server/lib/elastic-search/shared/elastic-search-avatar.ts b/server/lib/elastic-search/shared/elastic-search-avatar.ts deleted file mode 100644 index 0c6903a..0000000 --- a/server/lib/elastic-search/shared/elastic-search-avatar.ts +++ /dev/null @@ -1,77 +0,0 @@ -import { MappingProperty, PropertyName } from '@elastic/elasticsearch/lib/api/types' -import { ActorImage } from '@peertube/peertube-types' -import { buildUrl } from '../../../helpers/utils' - -function formatActorImageForAPI (image?: ActorImage) { - if (!image) return null - - return { - url: image.url, - path: image.path, - width: image.width, - createdAt: image.createdAt, - updatedAt: image.updatedAt - } -} - -function formatActorImagesForAPI (images?: ActorImage[], image?: ActorImage) { - // Does not exist in PeerTube < 4.2 - if (!images) { - if (!image) return [] - - return [ image ] - } - - return images.map(a => formatActorImageForAPI(a)) -} - -// --------------------------------------------------------------------------- - -function formatActorImageForDB (image: ActorImage, host: string) { - if (!image) return null - - return { - url: buildUrl(host, image.path), - path: image.path, - width: image.width, - createdAt: image.createdAt, - updatedAt: image.updatedAt - } -} - -function formatActorImagesForDB (images: ActorImage[], host: string) { - if (!images) return null - - return images.map(image => formatActorImageForDB(image, host)) -} - -// --------------------------------------------------------------------------- - -function buildActorImageMapping () { - return { - path: { - type: 'keyword' - }, - width: { - type: 'long' - }, - createdAt: { - type: 'date', - format: 'date_optional_time' - }, - updatedAt: { - type: 'date', - format: 'date_optional_time' - } - } as Record -} - -export { - formatActorImageForAPI, - formatActorImagesForAPI, - - formatActorImageForDB, - formatActorImagesForDB, - - buildActorImageMapping -} diff --git a/server/lib/elastic-search/shared/index.ts b/server/lib/elastic-search/shared/index.ts deleted file mode 100644 index 11842d2..0000000 --- a/server/lib/elastic-search/shared/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './elastic-search-actor' -export * from './elastic-search-avatar' -export * from './query-helpers' diff --git a/server/lib/elastic-search/shared/query-helpers.ts b/server/lib/elastic-search/shared/query-helpers.ts deleted file mode 100644 index dfd55ca..0000000 --- a/server/lib/elastic-search/shared/query-helpers.ts +++ /dev/null @@ -1,88 +0,0 @@ -import validator from 'validator' - -import { ELASTIC_SEARCH_QUERY } from '../../../initializers/constants' - -function addUUIDFilters (filters: any[], uuids: string[]) { - if (!filters) return - - const result = { - shortUUIDs: [] as string[], - uuids: [] as string[] - } - - for (const uuid of uuids) { - if (validator.isUUID(uuid)) result.uuids.push(uuid) - else result.shortUUIDs.push(uuid) - } - - filters.push({ - bool: { - should: [ - { - terms: { - uuid: result.uuids - } - }, - { - terms: { - shortUUID: result.shortUUIDs - } - } - ] - } - }) -} - -function buildMultiMatchBool (search: string, fieldsObject: { default: string[], phrase: string[] }) { - return { - must: [ - { - bool: { - should: [ - { - multi_match: { - query: search, - fields: fieldsObject.default, - fuzziness: ELASTIC_SEARCH_QUERY.FUZZINESS, - operator: ELASTIC_SEARCH_QUERY.OPERATOR, - minimum_should_match: ELASTIC_SEARCH_QUERY.MINIMUM_SHOULD_MATCH - } - }, - { - multi_match: { - query: search, - fields: fieldsObject.default, - operator: ELASTIC_SEARCH_QUERY.OPERATOR, - minimum_should_match: ELASTIC_SEARCH_QUERY.MINIMUM_SHOULD_MATCH, - type: 'cross_fields' - } - }, - { - multi_match: { - query: search, - fields: fieldsObject.phrase, - type: 'phrase' - } - } - ] - } - } - ], - should: [ - // Better score for exact search - { - multi_match: { - query: search, - fields: [ ...fieldsObject.default, ...fieldsObject.phrase ], - operator: ELASTIC_SEARCH_QUERY.OPERATOR, - minimum_should_match: ELASTIC_SEARCH_QUERY.MINIMUM_SHOULD_MATCH - } - } - ] - } -} - -export { - addUUIDFilters, - buildMultiMatchBool -} diff --git a/server/lib/indexers/channel-indexer.ts b/server/lib/indexers/channel-indexer.ts index 689f400..789a180 100644 --- a/server/lib/indexers/channel-indexer.ts +++ b/server/lib/indexers/channel-indexer.ts @@ -1,20 +1,19 @@ import { logger } from '../../helpers/logger' -import { CONFIG } from '../../initializers/constants' +import { CONFIG, SORTABLE_COLUMNS } from '../../initializers/constants' import { DBChannel, IndexableChannel } from '../../types/channel.model' -import { buildChannelsMapping, formatChannelForDB } from '../elastic-search/elastic-search-channels' +import { formatChannelForDB } from '../meilisearch/meilisearch-channels' import { getChannel } from '../requests/peertube-instance' import { AbstractIndexer } from './shared' export class ChannelIndexer extends AbstractIndexer { + protected readonly primaryKey = 'primaryKey' + protected readonly filterableAttributes = [ 'url', 'host', 'videosCount', 'ownerAccount.handle', 'handle' ] + protected readonly sortableAttributes = SORTABLE_COLUMNS.CHANNELS_SEARCH + // Keep the order, most important first + protected readonly searchableAttributes = [ 'name', 'displayName', 'ownerAccount.displayName', 'description' ] constructor () { - super(CONFIG.ELASTIC_SEARCH.INDEXES.CHANNELS, formatChannelForDB) - - this.indexQueue.drain(async () => { - logger.info('Refresh channels index.') - - await this.refreshIndex() - }) + super(CONFIG.MEILISEARCH.INDEXES.CHANNELS, formatChannelForDB) } async indexSpecificElement (host: string, name: string) { @@ -22,10 +21,6 @@ export class ChannelIndexer extends AbstractIndexer { + protected readonly primaryKey = 'uuid' + protected readonly filterableAttributes = [ 'uuid', 'host', 'videosLength' ] + protected readonly sortableAttributes = SORTABLE_COLUMNS.PLAYLISTS_SEARCH + // Keep the order, most important first + protected readonly searchableAttributes = [ 'displayName', 'videoChannel.displayName', 'ownerAccount.displayName', 'description' ] constructor () { - super(CONFIG.ELASTIC_SEARCH.INDEXES.PLAYLISTS, formatPlaylistForDB) + super(CONFIG.MEILISEARCH.INDEXES.PLAYLISTS, formatPlaylistForDB) } async indexSpecificElement (host: string, uuid: string) { // We don't need to index a specific element yet, since we have all playlist information in the list endpoint throw new Error('Not implemented') } - - buildMapping () { - return buildPlaylistsMapping() - } } diff --git a/server/lib/indexers/shared/abstract-indexer.ts b/server/lib/indexers/shared/abstract-indexer.ts index e314f87..8d2e032 100644 --- a/server/lib/indexers/shared/abstract-indexer.ts +++ b/server/lib/indexers/shared/abstract-indexer.ts @@ -1,11 +1,10 @@ import { AsyncQueue, queue } from 'async' import { inspect } from 'util' -import { MappingProperty, PropertyName } from '@elastic/elasticsearch/lib/api/types' import { logger } from '../../../helpers/logger' import { INDEXER_QUEUE_CONCURRENCY } from '../../../initializers/constants' -import { buildIndex, indexDocuments, refreshIndex } from '../../../lib/elastic-search/elastic-search-index' -import { removeFromHosts, removeNotExistingIdsFromHost } from '../../../lib/elastic-search/elastic-search-queries' import { IndexableDoc } from '../../../types/indexable-doc.model' +import { client } from '../../../helpers/meilisearch' +import { buildInValuesArray } from '../../meilisearch/meilisearch-queries' // identifier could be an uuid, an handle or a url for example export type QueueParam = { host: string, identifier: string } @@ -13,8 +12,14 @@ export type QueueParam = { host: string, identifier: string } export abstract class AbstractIndexer { protected readonly indexQueue: AsyncQueue + protected abstract readonly primaryKey: keyof DB + protected abstract readonly filterableAttributes: string[] + protected abstract readonly sortableAttributes: string[] + protected abstract readonly searchableAttributes: string[] + + protected readonly rankingRules: string[] + abstract indexSpecificElement (host: string, uuid: string): Promise - abstract buildMapping (): Record constructor ( protected readonly indexName: string, @@ -33,8 +38,25 @@ export abstract class AbstractIndexer { }, INDEXER_QUEUE_CONCURRENCY) } - initIndex () { - return buildIndex(this.indexName, this.buildMapping()) + async initIndex () { + const { results } = await client.getIndexes() + + if (results.some(r => r.uid === this.indexName)) { + logger.info(this.indexName + ' already exists, skipping configuration') + return + } + + logger.info('Creating and configuring index ' + this.indexName) + + await client.index(this.indexName).updateSearchableAttributes(this.searchableAttributes) + await client.index(this.indexName).updateFilterableAttributes(this.filterableAttributes) + await client.index(this.indexName).updateSortableAttributes(this.sortableAttributes) + await client.index(this.indexName).updateFaceting({ maxValuesPerFacet: 10_000 }) + await client.index(this.indexName).updatePagination({ maxTotalHits: 10_000 }) + + if (this.rankingRules) { + await client.index(this.indexName).updateRankingRules(this.rankingRules) + } } scheduleIndexation (host: string, identifier: string) { @@ -42,24 +64,22 @@ export abstract class AbstractIndexer { .catch(err => logger.error({ err: inspect(err) }, 'Cannot schedule indexation of %s for %s', identifier, host)) } - refreshIndex () { - return refreshIndex(this.indexName) - } - - removeNotExisting (host: string, existingIds: Set) { - return removeNotExistingIdsFromHost(this.indexName, host, existingIds) + removeNotExisting (host: string, existingPrimaryKeys: Set) { + return client.index(this.indexName).deleteDocuments({ + filter: `${this.primaryKey.toString()} NOT IN ${buildInValuesArray(Array.from(existingPrimaryKeys))} AND host = ${host}` + }) } removeFromHosts (hosts: string[]) { - return removeFromHosts(this.indexName, hosts) - } - - indexElements (elements: T[], replace = false) { - return indexDocuments({ - objects: elements, - formatter: v => this.formatterFn(v), - replace, - index: this.indexName + return client.index(this.indexName).deleteDocuments({ + filter: 'host IN ' + buildInValuesArray(Array.from(hosts)) }) } + + async indexElements (elements: T[]) { + const documents = elements.map(e => this.formatterFn(e)) + + const result = await client.index(this.indexName).updateDocuments(documents, { primaryKey: this.primaryKey.toString() }) + logger.debug(result, 'Indexed ' + documents.length + ' documents in ' + this.indexName) + } } diff --git a/server/lib/indexers/video-indexer.ts b/server/lib/indexers/video-indexer.ts index 42e0f3a..a9740ca 100644 --- a/server/lib/indexers/video-indexer.ts +++ b/server/lib/indexers/video-indexer.ts @@ -1,14 +1,54 @@ +import { AsyncQueue } from 'async' import { logger } from '../../helpers/logger' -import { CONFIG } from '../../initializers/constants' -import { DBVideo, IndexableVideo } from '../../types/video.model' -import { buildVideosMapping, formatVideoForDB } from '../elastic-search/elastic-search-videos' +import { AbstractIndexer, QueueParam } from './shared' +import { CONFIG, SORTABLE_COLUMNS } from '../../initializers/constants' +import { formatVideoForDB } from '../meilisearch/meilisearch-videos' import { getVideo } from '../requests/peertube-instance' -import { AbstractIndexer } from './shared' +import { DBVideo, IndexableVideo } from '../../types/video.model' export class VideoIndexer extends AbstractIndexer { + protected readonly indexQueue: AsyncQueue + protected readonly primaryKey = 'uuid' + protected readonly filterableAttributes = [ + 'uuid', + 'host', + 'account.handle', + 'account.host', + 'publishedAt', + 'originallyPublishedAt', + 'nsfw', + 'category.id', + 'licence.id', + 'language.id', + 'tags', + 'duration', + 'isLive' + ] + + protected readonly sortableAttributes = SORTABLE_COLUMNS.VIDEOS_SEARCH + + // Keep the order, most important first + protected readonly searchableAttributes = [ + 'name', + 'tags', + 'account.displayName', + 'channel.displayName', + 'description' + ] + + protected readonly rankingRules = [ + "words", + "typo", + "proximity", + "attribute", + "sort", + "exactness", + 'language:asc', + 'views:desc' + ] constructor () { - super(CONFIG.ELASTIC_SEARCH.INDEXES.VIDEOS, formatVideoForDB) + super(CONFIG.MEILISEARCH.INDEXES.VIDEOS, formatVideoForDB) } async indexSpecificElement (host: string, uuid: string) { @@ -16,10 +56,6 @@ export class VideoIndexer extends AbstractIndexer { logger.info('Indexing specific video %s of %s.', uuid, host) - return this.indexElements([ video ], true) - } - - buildMapping () { - return buildVideosMapping() + return this.indexElements([ video ]) } } diff --git a/server/lib/meilisearch/meilisearch-channels.ts b/server/lib/meilisearch/meilisearch-channels.ts new file mode 100644 index 0000000..8b5db3f --- /dev/null +++ b/server/lib/meilisearch/meilisearch-channels.ts @@ -0,0 +1,141 @@ +import { createHash } from 'crypto' +import { logger } from '../../helpers/logger' +import { client } from '../../helpers/meilisearch' +import { CONFIG } from '../../initializers/constants' +import { DBChannel, APIVideoChannel, IndexableChannel } from '../../types/channel.model' +import { ChannelsSearchQuery } from '../../types/search-query/channel-search.model' +import { buildInValuesArray, buildSort, extractSearchQueryResult } from './meilisearch-queries' +import { + formatActorImageForAPI, + formatActorImageForDB, + formatActorImagesForAPI, + formatActorImagesForDB +} from './shared/meilisearch-avatar' + +export async function queryChannels (search: ChannelsSearchQuery) { + const filter: string[] = [ 'videosCount != 0' ] + + if (search.host) filter.push(`host = '${search.host}'`) + if (search.blockedAccounts) filter.push(`ownerAccount.handle NOT IN ${buildInValuesArray(search.blockedAccounts)}`) + if (search.handles) filter.push(`handle IN ${buildInValuesArray(search.handles)}`) + if (search.blockedHosts) filter.push(`host NOT IN ${buildInValuesArray(search.blockedHosts)}`) + + logger.debug({ filter }, 'Will query Meilisearch for channels.') + + const result = await client.index(CONFIG.MEILISEARCH.INDEXES.CHANNELS).search(search.search, { + offset: search.start, + limit: search.count, + sort: buildSort(search.sort), + showRankingScore: true, + filter + }) + + return extractSearchQueryResult(result) +} + +export function formatChannelForAPI (c: DBChannel, fromHost?: string): APIVideoChannel { + return { + id: c.id, + + score: c._rankingScore, + + url: c.url, + name: c.name, + host: c.host, + followingCount: c.followingCount, + followersCount: c.followersCount, + createdAt: new Date(c.createdAt), + updatedAt: new Date(c.updatedAt), + + avatar: formatActorImageForAPI(c.avatar), + avatars: formatActorImagesForAPI(c.avatars, c.avatar), + + banner: formatActorImageForAPI(c.banner), + banners: formatActorImagesForAPI(c.banners, c.banner), + + displayName: c.displayName, + description: c.description, + support: c.support, + isLocal: fromHost === c.host, + + videosCount: c.videosCount || 0, + + ownerAccount: { + id: c.ownerAccount.id, + url: c.ownerAccount.url, + + displayName: c.ownerAccount.displayName, + description: c.ownerAccount.description, + name: c.ownerAccount.name, + host: c.ownerAccount.host, + followingCount: c.ownerAccount.followingCount, + followersCount: c.ownerAccount.followersCount, + createdAt: new Date(c.ownerAccount.createdAt), + updatedAt: new Date(c.ownerAccount.updatedAt), + + avatar: formatActorImageForAPI(c.ownerAccount.avatar), + avatars: formatActorImagesForAPI(c.ownerAccount.avatars, c.ownerAccount.avatar) + } + } +} + +export function formatChannelForDB (c: IndexableChannel): DBChannel { + return { + primaryKey: buildDBChannelPrimaryKey(c), + + id: c.id, + + name: c.name, + host: c.host, + url: c.url, + + avatar: formatActorImageForDB(c.avatar, c.host), + avatars: formatActorImagesForDB(c.avatars, c.host), + + banner: formatActorImageForDB(c.banner, c.host), + banners: formatActorImagesForDB(c.banners, c.host), + + displayName: c.displayName, + + indexedAt: new Date().getTime(), + createdAt: new Date(c.createdAt).getTime(), + updatedAt: new Date(c.updatedAt).getTime(), + + followingCount: c.followingCount, + followersCount: c.followersCount, + + description: c.description, + support: c.support, + videosCount: c.videosCount, + + handle: `${c.name}@${c.host}`, + + ownerAccount: { + id: c.ownerAccount.id, + url: c.ownerAccount.url, + + displayName: c.ownerAccount.displayName, + description: c.ownerAccount.description, + name: c.ownerAccount.name, + host: c.ownerAccount.host, + followingCount: c.ownerAccount.followingCount, + followersCount: c.ownerAccount.followersCount, + createdAt: new Date(c.ownerAccount.createdAt).getTime(), + updatedAt: new Date(c.ownerAccount.updatedAt).getTime(), + + handle: `${c.ownerAccount.name}@${c.ownerAccount.host}`, + + avatar: formatActorImageForDB(c.ownerAccount.avatar, c.ownerAccount.host), + avatars: formatActorImagesForDB(c.ownerAccount.avatars, c.ownerAccount.host) + } + } +} + +export function buildDBChannelPrimaryKey (c: { id: number, host: string }) { + return `${c.id}_${md5(c.host)}` +} + +// Collisions are fine, we just want to generate a primary key in an efficient way +function md5 (value: string) { + return createHash('md5').update(value).digest('hex') +} diff --git a/server/lib/elastic-search/elastic-search-instances.ts b/server/lib/meilisearch/meilisearch-instances.ts similarity index 58% rename from server/lib/elastic-search/elastic-search-instances.ts rename to server/lib/meilisearch/meilisearch-instances.ts index 44555ae..db9d8ec 100644 --- a/server/lib/elastic-search/elastic-search-instances.ts +++ b/server/lib/meilisearch/meilisearch-instances.ts @@ -1,8 +1,6 @@ -import { AggregationsStringTermsAggregate } from '@elastic/elasticsearch/lib/api/types' -import { elasticSearch } from '../../helpers/elastic-search' import { CONFIG } from '../../initializers/constants' import { listIndexInstancesHost } from '../requests/instances-index' -import { extractBucketsFromAggregation } from './elastic-search-queries' +import { client } from '../../helpers/meilisearch' async function buildInstanceHosts () { let indexHosts = await listIndexInstancesHost() @@ -30,28 +28,15 @@ export { async function listDBInstances () { const setResult = new Set() const indexes = [ - CONFIG.ELASTIC_SEARCH.INDEXES.VIDEOS, - CONFIG.ELASTIC_SEARCH.INDEXES.CHANNELS + CONFIG.MEILISEARCH.INDEXES.VIDEOS, + CONFIG.MEILISEARCH.INDEXES.CHANNELS ] for (const index of indexes) { - const res = await elasticSearch.search>({ - index, - body: { - size: 0, - aggs: { - hosts: { - terms: { - size: 5000, - field: 'host' - } - } - } - } - }) + const result = await client.index(index).searchForFacetValues({ facetName: 'host' }) - for (const b of extractBucketsFromAggregation(res.aggregations.hosts.buckets)) { - setResult.add(b.key) + for (const b of result.facetHits) { + setResult.add(b.value) } } diff --git a/server/lib/meilisearch/meilisearch-playlists.ts b/server/lib/meilisearch/meilisearch-playlists.ts new file mode 100644 index 0000000..d12c593 --- /dev/null +++ b/server/lib/meilisearch/meilisearch-playlists.ts @@ -0,0 +1,108 @@ +import { logger } from '../../helpers/logger' +import { client } from '../../helpers/meilisearch' +import { buildUrl } from '../../helpers/utils' +import { CONFIG } from '../../initializers/constants' +import { DBPlaylist, APIPlaylist, IndexablePlaylist } from '../../types/playlist.model' +import { PlaylistsSearchQuery } from '../../types/search-query/playlist-search.model' +import { buildInValuesArray, buildSort, extractSearchQueryResult } from './meilisearch-queries' +import { addUUIDFilters } from './shared' +import { formatActorForDB, formatActorSummaryForAPI } from './shared/meilisearch-actor' + +export async function queryPlaylists (search: PlaylistsSearchQuery) { + const filter: string[] = [ + 'videosLength != 0' + ] + + if (search.host) filter.push(`ownerAccount.host = '${search.host}'`) + if (search.blockedAccounts) filter.push(`ownerAccount.handle NOT IN ${buildInValuesArray(search.blockedAccounts)}`) + if (search.blockedHosts) filter.push(`host NOT IN ${buildInValuesArray(search.blockedHosts)}`) + if (search.uuids) addUUIDFilters(filter, search.uuids) + + logger.debug({ filter }, 'Will query Meilisearch for playlists.') + + const result = await client.index(CONFIG.MEILISEARCH.INDEXES.PLAYLISTS).search(search.search, { + offset: search.start, + limit: search.count, + sort: buildSort(search.sort), + showRankingScore: true, + filter + }) + + return extractSearchQueryResult(result) +} + +export function formatPlaylistForAPI (p: DBPlaylist, fromHost?: string): APIPlaylist { + return { + id: p.id, + uuid: p.uuid, + shortUUID: p.shortUUID, + + score: p._rankingScore, + + isLocal: fromHost === p.host, + + url: p.url, + + displayName: p.displayName, + description: p.description, + + privacy: { + id: p.privacy.id, + label: p.privacy.label + }, + + videosLength: p.videosLength, + + type: { + id: p.type.id, + label: p.type.label + }, + + thumbnailPath: p.thumbnailPath, + thumbnailUrl: buildUrl(p.host, p.thumbnailPath), + + embedPath: p.embedPath, + embedUrl: buildUrl(p.host, p.embedPath), + + createdAt: new Date(p.createdAt), + updatedAt: new Date(p.updatedAt), + + ownerAccount: formatActorSummaryForAPI(p.ownerAccount), + videoChannel: formatActorSummaryForAPI(p.videoChannel) + } +} + +export function formatPlaylistForDB (p: IndexablePlaylist): DBPlaylist { + return { + id: p.id, + uuid: p.uuid, + shortUUID: p.shortUUID, + + indexedAt: new Date().getTime(), + createdAt: new Date(p.createdAt).getTime(), + updatedAt: new Date(p.updatedAt).getTime(), + + host: p.host, + url: p.url, + + displayName: p.displayName, + description: p.description, + + thumbnailPath: p.thumbnailPath, + embedPath: p.embedPath, + + type: { + id: p.type.id, + label: p.type.label + }, + privacy: { + id: p.privacy.id, + label: p.privacy.label + }, + + videosLength: p.videosLength, + + ownerAccount: formatActorForDB(p.ownerAccount), + videoChannel: formatActorForDB(p.videoChannel) + } +} diff --git a/server/lib/meilisearch/meilisearch-queries.ts b/server/lib/meilisearch/meilisearch-queries.ts new file mode 100644 index 0000000..abd97b4 --- /dev/null +++ b/server/lib/meilisearch/meilisearch-queries.ts @@ -0,0 +1,27 @@ +import { SearchResponse } from 'meilisearch' + +export function extractSearchQueryResult (result: SearchResponse) { + return { + total: result.estimatedTotalHits, + data: result.hits + } +} + +export function buildSort (value: string) { + if (value === '-match') return [ '_rankingScore:desc' ] + if (value === 'match') return [ '_rankingScore:asc' ] + + if (value.substring(0, 1) === '-') { + return [ `${value.substring(1)}:desc`, '_rankingScore:desc' ] + } + + return [ `${value}:asc`, '_rankingScore:desc' ] +} + +export function buildInQuery (term: string, values: string[] | number[]) { + return `${term} IN ${buildInValuesArray(values)}` +} + +export function buildInValuesArray (values: string[] | number[]) { + return '[' + values.map(v => `'${v}'`).join(',') + ']' +} diff --git a/server/lib/meilisearch/meilisearch-videos.ts b/server/lib/meilisearch/meilisearch-videos.ts new file mode 100644 index 0000000..483b519 --- /dev/null +++ b/server/lib/meilisearch/meilisearch-videos.ts @@ -0,0 +1,226 @@ +import { exists } from '../../helpers/custom-validators/misc' +import { logger } from '../../helpers/logger' +import { client } from '../../helpers/meilisearch' +import { buildUrl } from '../../helpers/utils' +import { CONFIG } from '../../initializers/constants' +import { VideosSearchQuery } from '../../types/search-query/video-search.model' +import { DBVideo, DBVideoDetails, APIVideo, IndexableVideo, IndexableVideoDetails } from '../../types/video.model' +import { buildInQuery, buildInValuesArray, buildSort, extractSearchQueryResult } from './meilisearch-queries' +import { addUUIDFilters } from './shared' +import { formatActorForDB, formatActorSummaryForAPI } from './shared/meilisearch-actor' + +export async function queryVideos (search: VideosSearchQuery) { + const filter: string[] = [] + + if (search.blockedAccounts) filter.push(`account.handle NOT IN ${buildInValuesArray(search.blockedAccounts)}`) + if (search.blockedHosts) filter.push(`host NOT IN ${buildInValuesArray(search.blockedHosts)}`) + + if (search.startDate) filter.push(`publishedAt >= ${new Date(search.startDate).getTime()}`) + if (search.endDate) filter.push(`publishedAt <= ${new Date(search.endDate).getTime()}`) + + if (search.originallyPublishedStartDate) { + filter.push(`originallyPublishedAt >= ${new Date(search.originallyPublishedStartDate).getTime()}`) + } + if (search.originallyPublishedEndDate) { + filter.push(`originallyPublishedAt <= ${new Date(search.originallyPublishedEndDate).getTime()}`) + } + + if (search.categoryOneOf) filter.push(`category.id IN ${buildInValuesArray(search.categoryOneOf)}`) + if (search.licenceOneOf) filter.push(`licence.id IN ${buildInValuesArray(search.licenceOneOf)}`) + if (search.languageOneOf) filter.push(`language.id IN ${buildInValuesArray(search.languageOneOf)}`) + + if (search.durationMin) filter.push(`duration >= ${search.durationMin}`) + if (search.durationMax) filter.push(`duration <= ${search.durationMax}`) + + if (search.nsfw && search.nsfw !== 'both') filter.push(`nsfw = ${search.nsfw}`) + + if (exists(search.isLive)) filter.push(`isLive = ${search.isLive}`) + if (search.host) filter.push(`account.host = '${search.host}'`) + if (search.uuids) addUUIDFilters(filter, search.uuids) + + if (search.tagsOneOf) { + const tagsOneOf = search.tagsOneOf.map(t => t.toLowerCase()) + filter.push(`tags IN ${buildInValuesArray(tagsOneOf)}`) + } + + if (search.tagsAllOf) { + const tagsAllOf = search.tagsAllOf.map(t => t.toLowerCase()) + const clause = tagsAllOf.map(tag => `tags = '${tag}'`).join(' AND ') + + filter.push(clause) + } + + logger.debug({ filter }, 'Will query Meilisearch for videos.') + + const result = await client.index(CONFIG.MEILISEARCH.INDEXES.VIDEOS).search(search.search, { + offset: search.start, + limit: search.count, + sort: buildSort(search.sort), + showRankingScore: true, + filter: [ + ...filter, + + `language.id IN ${buildInValuesArray(search.boostLanguages)} OR language.id IS NULL` + ] + }) + + return extractSearchQueryResult(result) +} + +export async function getVideosUpdatedAt (uuids: string[]): Promise<{ updatedAt: number, uuid: string }[]> { + const result = await client.index(CONFIG.MEILISEARCH.INDEXES.VIDEOS).getDocuments({ + fields: [ 'updatedAt', 'uuid' ], + filter: [ buildInQuery('uuid', uuids) ] + }) + + return result.results +} + +export function formatVideoForDB (v: IndexableVideo | IndexableVideoDetails): DBVideo | DBVideoDetails { + const video = { + id: v.id, + uuid: v.uuid, + shortUUID: v.shortUUID, + + indexedAt: new Date().getTime(), + createdAt: new Date(v.createdAt).getTime(), + updatedAt: new Date(v.updatedAt).getTime(), + publishedAt: new Date(v.publishedAt).getTime(), + originallyPublishedAt: new Date(v.originallyPublishedAt).getTime(), + + category: { + id: v.category.id, + label: v.category.label + }, + licence: { + id: v.licence.id, + label: v.licence.label + }, + language: { + id: v.language.id, + label: v.language.label + }, + privacy: { + id: v.privacy.id, + label: v.privacy.label + }, + + name: v.name, + + truncatedDescription: v.truncatedDescription, + description: v.description, + + waitTranscoding: v.waitTranscoding, + + duration: v.duration, + + thumbnailPath: v.thumbnailPath, + previewPath: v.previewPath, + embedPath: v.embedPath, + + views: v.views, + viewers: v.viewers, + likes: v.likes, + dislikes: v.dislikes, + + isLive: v.isLive || false, + nsfw: v.nsfw, + + host: v.host, + url: v.url, + + files: v.files, + streamingPlaylists: v.streamingPlaylists, + + tags: (v as IndexableVideoDetails).tags ? (v as IndexableVideoDetails).tags : undefined, + + account: formatActorForDB(v.account), + channel: formatActorForDB(v.channel) + } + + if (isVideoDetails(v)) { + return { + ...video, + + trackerUrls: v.trackerUrls, + + descriptionPath: v.descriptionPath, + + support: v.support, + + commentsEnabled: v.commentsEnabled, + downloadEnabled: v.downloadEnabled + } + } + + return video +} + +export function formatVideoForAPI (v: DBVideoDetails, fromHost?: string): APIVideo { + return { + id: v.id, + uuid: v.uuid, + shortUUID: v.shortUUID, + + score: v._rankingScore, + + createdAt: new Date(v.createdAt), + updatedAt: new Date(v.updatedAt), + publishedAt: new Date(v.publishedAt), + originallyPublishedAt: new Date(v.originallyPublishedAt), + + category: { + id: v.category.id, + label: v.category.label + }, + licence: { + id: v.licence.id, + label: v.licence.label + }, + language: { + id: v.language.id, + label: v.language.label + }, + privacy: { + id: v.privacy.id, + label: v.privacy.label + }, + + name: v.name, + description: v.description, + truncatedDescription: v.truncatedDescription, + duration: v.duration, + + tags: v.tags, + + thumbnailPath: v.thumbnailPath, + thumbnailUrl: buildUrl(v.host, v.thumbnailPath), + + previewPath: v.previewPath, + previewUrl: buildUrl(v.host, v.previewPath), + + embedPath: v.embedPath, + embedUrl: buildUrl(v.host, v.embedPath), + + url: v.url, + + isLocal: fromHost && fromHost === v.host, + + views: v.views, + viewers: v.viewers, + likes: v.likes, + dislikes: v.dislikes, + + isLive: v.isLive, + nsfw: v.nsfw, + + account: formatActorSummaryForAPI(v.account), + channel: formatActorSummaryForAPI(v.channel) + } +} + +// --------------------------------------------------------------------------- + +export function isVideoDetails (video: IndexableVideo | IndexableVideoDetails): video is IndexableVideoDetails { + return (video as IndexableVideoDetails).commentsEnabled !== undefined +} diff --git a/server/lib/meilisearch/shared/index.ts b/server/lib/meilisearch/shared/index.ts new file mode 100644 index 0000000..35f798d --- /dev/null +++ b/server/lib/meilisearch/shared/index.ts @@ -0,0 +1,3 @@ +export * from './meilisearch-actor' +export * from './meilisearch-avatar' +export * from './query-helpers' diff --git a/server/lib/meilisearch/shared/meilisearch-actor.ts b/server/lib/meilisearch/shared/meilisearch-actor.ts new file mode 100644 index 0000000..a6cf471 --- /dev/null +++ b/server/lib/meilisearch/shared/meilisearch-actor.ts @@ -0,0 +1,32 @@ +import { AccountSummary, VideoChannelSummary } from '@peertube/peertube-types' +import { formatActorImageForAPI, formatActorImageForDB, formatActorImagesForAPI, formatActorImagesForDB } from './meilisearch-avatar' +import { DBAccountSummary } from '../../../types/account.model' +import { DBChannelSummary } from '../../../types/channel.model' + +export function formatActorSummaryForAPI (actor: DBAccountSummary | DBChannelSummary) { + return { + id: actor.id, + name: actor.name, + displayName: actor.displayName, + url: actor.url, + host: actor.host, + + avatar: formatActorImageForAPI(actor.avatar), + avatars: formatActorImagesForAPI(actor.avatars, actor.avatar) + } +} + +export function formatActorForDB (actor: AccountSummary | VideoChannelSummary) { + return { + id: actor.id, + name: actor.name, + displayName: actor.displayName, + url: actor.url, + host: actor.host, + + handle: `${actor.name}@${actor.host}`, + + avatar: formatActorImageForDB(actor.avatar, actor.host), + avatars: formatActorImagesForDB(actor.avatars, actor.host) + } +} diff --git a/server/lib/meilisearch/shared/meilisearch-avatar.ts b/server/lib/meilisearch/shared/meilisearch-avatar.ts new file mode 100644 index 0000000..dcf63f2 --- /dev/null +++ b/server/lib/meilisearch/shared/meilisearch-avatar.ts @@ -0,0 +1,46 @@ +import { ActorImage } from '@peertube/peertube-types' +import { buildUrl } from '../../../helpers/utils' +import { DBActorImage } from '../../../types/actor.model' + +export function formatActorImageForAPI (image?: DBActorImage) { + if (!image) return null + + return { + url: image.url, + path: image.path, + width: image.width, + createdAt: new Date(image.createdAt), + updatedAt: new Date(image.updatedAt) + } +} + +export function formatActorImagesForAPI (images?: DBActorImage[], image?: DBActorImage) { + // Does not exist in PeerTube < 4.2 + if (!images) { + if (!image) return [] + + return [ formatActorImageForAPI(image) ] + } + + return images.map(a => formatActorImageForAPI(a)) +} + +// --------------------------------------------------------------------------- + +export function formatActorImageForDB (image: ActorImage, host: string) { + if (!image) return null + + return { + url: buildUrl(host, image.path), + path: image.path, + width: image.width, + createdAt: new Date(image.createdAt).getTime(), + updatedAt: new Date(image.updatedAt).getTime() + } +} + +export function formatActorImagesForDB (images: ActorImage[], host: string) { + if (!images) return null + + return images.map(image => formatActorImageForDB(image, host)) +} diff --git a/server/lib/meilisearch/shared/query-helpers.ts b/server/lib/meilisearch/shared/query-helpers.ts new file mode 100644 index 0000000..de9dfe3 --- /dev/null +++ b/server/lib/meilisearch/shared/query-helpers.ts @@ -0,0 +1,22 @@ +import validator from 'validator' +import { buildInQuery } from '../meilisearch-queries' + +export function addUUIDFilters (filters: string[], uuids: string[]) { + if (!filters || filters.length === 0) return + + const result = { + shortUUIDs: [] as string[], + uuids: [] as string[] + } + + for (const uuid of uuids) { + if (validator.isUUID(uuid)) result.uuids.push(uuid) + else result.shortUUIDs.push(uuid) + } + + const parts: string[] = [] + if (result.uuids.length !== 0) parts.push(buildInQuery('uuid', result.uuids)) + if (result.shortUUIDs.length !== 0) parts.push(buildInQuery('shortUUID', result.shortUUIDs)) + + filters.push(parts.join(' OR ')) +} diff --git a/server/lib/requests/peertube-instance.ts b/server/lib/requests/peertube-instance.ts index f7f032f..3531cd9 100644 --- a/server/lib/requests/peertube-instance.ts +++ b/server/lib/requests/peertube-instance.ts @@ -1,10 +1,10 @@ -import { IndexablePlaylist } from 'server/types/playlist.model' import { ResultList, Video, VideoChannel, VideoDetails, VideoPlaylist, VideosCommonQuery } from '@peertube/peertube-types' import { doRequestWithRetries } from '../../helpers/requests' import { INDEXER_COUNT, REQUESTS } from '../../initializers/constants' import { IndexableChannel } from '../../types/channel.model' import { IndexableDoc } from '../../types/indexable-doc.model' import { IndexableVideo } from '../../types/video.model' +import { IndexablePlaylist } from '../../types/playlist.model' async function getVideo (host: string, uuid: string): Promise { const url = 'https://' + host + '/api/v1/videos/' + uuid @@ -92,7 +92,6 @@ async function getPlaylistsOf (host: string, handle: string, start: number): Pro function prepareVideoForDB (video: T, host: string): T & IndexableDoc { return Object.assign(video, { - elasticSearchId: host + video.id, host, url: 'https://' + host + '/videos/watch/' + video.uuid }) @@ -100,7 +99,6 @@ function prepareVideoForDB (video: T, host: string): T & Index function prepareChannelForDB (channel: VideoChannel, host: string, videosCount: number): IndexableChannel { return Object.assign(channel, { - elasticSearchId: host + channel.id, host, videosCount, url: 'https://' + host + '/video-channels/' + channel.name @@ -109,7 +107,6 @@ function prepareChannelForDB (channel: VideoChannel, host: string, videosCount: function preparePlaylistForDB (playlist: VideoPlaylist, host: string): IndexablePlaylist { return Object.assign(playlist, { - elasticSearchId: host + playlist.id, host, url: 'https://' + host + '/videos/watch/playlist/' + playlist.uuid }) diff --git a/server/lib/schedulers/indexation-scheduler.ts b/server/lib/schedulers/indexation-scheduler.ts index a66e5c7..620000d 100644 --- a/server/lib/schedulers/indexation-scheduler.ts +++ b/server/lib/schedulers/indexation-scheduler.ts @@ -1,15 +1,17 @@ import Bluebird from 'bluebird' -import { IndexablePlaylist } from 'server/types/playlist.model' import { inspect } from 'util' import { logger } from '../../helpers/logger' import { INDEXER_HOST_CONCURRENCY, INDEXER_COUNT, INDEXER_LIMIT, SCHEDULER_INTERVALS_MS } from '../../initializers/constants' import { IndexableVideo } from '../../types/video.model' -import { buildInstanceHosts } from '../elastic-search/elastic-search-instances' +import { buildInstanceHosts } from '../meilisearch/meilisearch-instances' import { ChannelIndexer } from '../indexers/channel-indexer' import { PlaylistIndexer } from '../indexers/playlist-indexer' import { VideoIndexer } from '../indexers/video-indexer' import { getPlaylistsOf, getVideos } from '../requests/peertube-instance' import { AbstractScheduler } from './abstract-scheduler' +import { IndexablePlaylist } from '../../types/playlist.model' +import { buildDBChannelPrimaryKey } from '../meilisearch/meilisearch-channels' +import { getVideosUpdatedAt } from '../meilisearch/meilisearch-videos' export class IndexationScheduler extends AbstractScheduler { @@ -70,17 +72,13 @@ export class IndexationScheduler extends AbstractScheduler { } }, { concurrency: INDEXER_HOST_CONCURRENCY }) - for (const o of this.indexers) { - await o.refreshIndex() - } - logger.info('Indexer ended.') } private async indexHost (host: string) { const channelsToSync = new Set() - const existingChannelsId = new Set() - const existingVideosId = new Set() + const existingChannelsId = new Set() + const existingVideosId = new Set() let videos: IndexableVideo[] = [] let start = 0 @@ -96,21 +94,27 @@ export class IndexationScheduler extends AbstractScheduler { start += videos.length if (videos.length !== 0) { - const { created } = await this.videoIndexer.indexElements(videos) + const videosFromDB = await getVideosUpdatedAt(videos.map(v => v.uuid)) + + await this.videoIndexer.indexElements(videos) logger.debug('Indexed %d videos from %s.', videos.length, host) - // Fetch complete video foreach created video (to get tags) - for (const c of created) { - this.videoIndexer.scheduleIndexation(host, c.uuid) + // Fetch complete video foreach created video (to get tags) if needed + for (const video of videos) { + const videoDB = videosFromDB.find(v => v.uuid === video.uuid) + + if (!videoDB || videoDB.updatedAt !== new Date(video.updatedAt).getTime()) { + this.videoIndexer.scheduleIndexation(host, video.uuid) + } } } for (const video of videos) { channelsToSync.add(video.channel.name) - existingChannelsId.add(video.channel.id) - existingVideosId.add(video.id) + existingChannelsId.add(buildDBChannelPrimaryKey(video.channel)) + existingVideosId.add(video.uuid) } } while (videos.length === INDEXER_COUNT && start < INDEXER_LIMIT) @@ -120,6 +124,7 @@ export class IndexationScheduler extends AbstractScheduler { this.channelIndexer.scheduleIndexation(host, c) } + logger.info('Removing non-existing channels and videos from ' + host) await this.channelIndexer.removeNotExisting(host, existingChannelsId) await this.videoIndexer.removeNotExisting(host, existingVideosId) @@ -127,7 +132,7 @@ export class IndexationScheduler extends AbstractScheduler { } private async indexPlaylists (host: string, channelHandles: string[]) { - const existingPlaylistsId = new Set() + const existingPlaylistsId = new Set() logger.info('Adding playlist data from %s.', host) @@ -150,7 +155,7 @@ export class IndexationScheduler extends AbstractScheduler { } for (const playlist of playlists) { - existingPlaylistsId.add(playlist.id) + existingPlaylistsId.add(playlist.uuid) } } while (playlists.length === INDEXER_COUNT && start < INDEXER_LIMIT) } diff --git a/server/types/account.model.ts b/server/types/account.model.ts new file mode 100644 index 0000000..2eee16c --- /dev/null +++ b/server/types/account.model.ts @@ -0,0 +1,25 @@ +import { Account, AccountSummary } from '@peertube/peertube-types' +import { DBActorImage } from './actor.model' + +type IgnoredAccountFields = 'createdAt' | 'updatedAt' | 'avatar' | 'avatars' + +interface DBAccountShared { + handle: string + + avatar: DBActorImage + avatars: DBActorImage[] +} + +export interface DBAccount extends Omit { + handle: string + + avatar: DBActorImage + avatars: DBActorImage[] + + createdAt: number + updatedAt: number +} + +export interface DBAccountSummary extends DBAccountShared, Omit { + +} diff --git a/server/types/actor.model.ts b/server/types/actor.model.ts index 4cbbe54..2a0f31c 100644 --- a/server/types/actor.model.ts +++ b/server/types/actor.model.ts @@ -1,11 +1,7 @@ import { ActorImage } from '@peertube/peertube-types' -export type AdditionalActorAttributes = { - handle: string +export type DBActorImage = Omit & { url: string - - avatar: ActorImageExtended - avatars: ActorImageExtended[] + createdAt: number + updatedAt: number } - -export type ActorImageExtended = ActorImage & { url: string } diff --git a/server/types/channel.model.ts b/server/types/channel.model.ts index e8c7066..4e60d15 100644 --- a/server/types/channel.model.ts +++ b/server/types/channel.model.ts @@ -1,33 +1,42 @@ -import { Account, VideoChannel, VideoChannelSummary } from '@peertube/peertube-types' -import { ActorImageExtended, AdditionalActorAttributes } from './actor.model' +import { VideoChannel, VideoChannelSummary } from '@peertube/peertube-types' +import { DBActorImage } from './actor.model' import { IndexableDoc } from './indexable-doc.model' +import { DBAccount } from './account.model' export interface IndexableChannel extends VideoChannel, IndexableDoc { - url: string } -export interface DBChannel extends Omit { - indexedAt: Date +interface DBChannelShared { handle: string - url: string - ownerAccount?: Account & AdditionalActorAttributes - - avatar: ActorImageExtended - avatars: ActorImageExtended[] - - banner: ActorImageExtended - banners: ActorImageExtended[] - - score?: number + avatar: DBActorImage + avatars: DBActorImage[] } -export interface DBChannelSummary extends VideoChannelSummary { - indexedAt: Date +type IgnoredChannelFields = 'ownerAccount' | 'isLocal' | 'createdAt' | 'updatedAt' | 'avatar' | 'avatars' | 'banner' | 'banners' + +export interface DBChannel extends DBChannelShared, Omit { + indexedAt: number + + primaryKey: string + + ownerAccount?: DBAccount + + banner: DBActorImage + banners: DBActorImage[] + + createdAt: number + updatedAt: number + + _rankingScore?: number +} + +export interface DBChannelSummary extends DBChannelShared, Omit { + } // Results from the search API -export interface EnhancedVideoChannel extends VideoChannel { +export interface APIVideoChannel extends VideoChannel { videosCount: number score: number diff --git a/server/types/indexable-doc.model.ts b/server/types/indexable-doc.model.ts index a7da3d9..12df230 100644 --- a/server/types/indexable-doc.model.ts +++ b/server/types/indexable-doc.model.ts @@ -1,5 +1,4 @@ export interface IndexableDoc { - elasticSearchId: string host: string url: string } diff --git a/server/types/playlist.model.ts b/server/types/playlist.model.ts index 67e43c5..06ea3f1 100644 --- a/server/types/playlist.model.ts +++ b/server/types/playlist.model.ts @@ -1,24 +1,26 @@ -import { AccountSummary, VideoChannelSummary, VideoPlaylist } from '@peertube/peertube-types' -import { AdditionalActorAttributes } from './actor.model' +import { VideoPlaylist } from '@peertube/peertube-types' import { IndexableDoc } from './indexable-doc.model' +import { DBAccountSummary } from './account.model' +import { DBChannelSummary } from './channel.model' export interface IndexablePlaylist extends VideoPlaylist, IndexableDoc { - url: string } -export interface DBPlaylist extends Omit { - indexedAt: Date +export interface DBPlaylist extends Omit { + indexedAt: number + createdAt: number + updatedAt: number host: string // Added by the query - score?: number + _rankingScore?: number - ownerAccount: AccountSummary & AdditionalActorAttributes - videoChannel: VideoChannelSummary & AdditionalActorAttributes + ownerAccount: DBAccountSummary + videoChannel: DBChannelSummary } // Results from the search API -export interface EnhancedPlaylist extends VideoPlaylist { +export interface APIPlaylist extends VideoPlaylist { score: number } diff --git a/server/types/video.model.ts b/server/types/video.model.ts index 077b3b2..cc6ff65 100644 --- a/server/types/video.model.ts +++ b/server/types/video.model.ts @@ -1,6 +1,7 @@ -import { Account, AccountSummary, Video, VideoChannel, VideoChannelSummary, VideoDetails } from '@peertube/peertube-types' -import { AdditionalActorAttributes } from './actor.model' +import { Video, VideoDetails } from '@peertube/peertube-types' import { IndexableDoc } from './indexable-doc.model' +import { DBChannel, DBChannelSummary } from './channel.model' +import { DBAccount, DBAccountSummary } from './account.model' export interface IndexableVideo extends Video, IndexableDoc { } @@ -8,28 +9,34 @@ export interface IndexableVideo extends Video, IndexableDoc { export interface IndexableVideoDetails extends VideoDetails, IndexableDoc { } -export interface DBVideoDetails extends Omit { - indexedAt: Date +type IgnoredVideoFields = 'isLocal' | 'createdAt' | 'updatedAt' | 'publishedAt' | 'originallyPublishedAt' | 'channel' | 'account' + +interface DBVideoShared { + indexedAt: number + + createdAt: number + updatedAt: number + publishedAt: number + originallyPublishedAt: number + host: string url: string - - account: Account & AdditionalActorAttributes - channel: VideoChannel & AdditionalActorAttributes - - score?: number } -export interface DBVideo extends Omit { - indexedAt: Date - host: string - url: string +export interface DBVideo extends Omit, DBVideoShared { + account: DBAccountSummary + channel: DBChannelSummary +} - account: AccountSummary & AdditionalActorAttributes - channel: VideoChannelSummary & AdditionalActorAttributes +export interface DBVideoDetails extends Omit, DBVideoShared { + account: DBAccount + channel: DBChannel + + _rankingScore?: number } // Results from the search API -export interface EnhancedVideo extends Video { +export interface APIVideo extends Video { tags: VideoDetails['tags'] score: number diff --git a/tsconfig.json b/tsconfig.json index 298e625..3dc1bdf 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -16,13 +16,11 @@ ], "types": [ "node" - ], - "baseUrl": "." + ] }, "exclude": [ "node_modules", "dist", - "client", - "PeerTube" + "client" ] } diff --git a/yarn.lock b/yarn.lock index ba5da92..b300ebf 100644 --- a/yarn.lock +++ b/yarn.lock @@ -980,26 +980,6 @@ enabled "2.0.x" kuler "^2.0.0" -"@elastic/elasticsearch@^8.2.1": - version "8.5.0" - resolved "https://registry.yarnpkg.com/@elastic/elasticsearch/-/elasticsearch-8.5.0.tgz#407aee0950a082ee76735a567f2571cf4301d4ea" - integrity sha512-iOgr/3zQi84WmPhAplnK2W13R89VXD2oc6WhlQmH3bARQwmI+De23ZJKBEn7bvuG/AHMAqasPXX7uJIiJa2MqQ== - dependencies: - "@elastic/transport" "^8.2.0" - tslib "^2.4.0" - -"@elastic/transport@^8.2.0": - version "8.3.1" - resolved "https://registry.yarnpkg.com/@elastic/transport/-/transport-8.3.1.tgz#e7569d7df35b03108ea7aa886113800245faa17f" - integrity sha512-jv/Yp2VLvv5tSMEOF8iGrtL2YsYHbpf4s+nDsItxUTLFTzuJGpnsB/xBlfsoT2kAYEnWHiSJuqrbRcpXEI/SEQ== - dependencies: - debug "^4.3.4" - hpagent "^1.0.0" - ms "^2.1.3" - secure-json-parse "^2.4.0" - tslib "^2.4.0" - undici "^5.5.1" - "@eslint/eslintrc@^1.3.3": version "1.3.3" resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-1.3.3.tgz#2b044ab39fdfa75b4688184f9e573ce3c5b0ff95" @@ -1202,7 +1182,7 @@ resolved "https://registry.yarnpkg.com/@types/async/-/async-3.2.16.tgz#7473aa015cf8a60584a94dc79b9203d465c32b41" integrity sha512-jnlGp5Z/cAZ7JVYyLnSDuYJ+YyYm0o2yzL8Odv6ckWmGMow3j/P/wgfziybB044cXXA93lEuymJyxVR8Iz2amQ== -"@types/bluebird@*", "@types/bluebird@^3.5.33": +"@types/bluebird@^3.5.33": version "3.5.38" resolved "https://registry.yarnpkg.com/@types/bluebird/-/bluebird-3.5.38.tgz#7a671e66750ccd21c9fc9d264d0e1e5330bc9908" integrity sha512-yR/Kxc0dd4FfwtEoLZMoqJbM/VE/W7hXn/MIjb+axcwag0iFmSPK7OBUZq1YWLynJUoWQkfUrI7T0HDqGApNSg== @@ -1242,13 +1222,6 @@ dependencies: "@types/node" "*" -"@types/continuation-local-storage@*": - version "3.2.4" - resolved "https://registry.yarnpkg.com/@types/continuation-local-storage/-/continuation-local-storage-3.2.4.tgz#655c8ffd9327aa60fb8ae773a5f2efbc973a7cbb" - integrity sha512-OT32vCVMymU1JMPKDeY0lX3cduAr0Pm/VwIbxygMeDS4lRcv57qYXn9bMwBRcRnEpXKBb/r4xYaZCARTZllP0A== - dependencies: - "@types/node" "*" - "@types/cookie@^0.4.1": version "0.4.1" resolved "https://registry.yarnpkg.com/@types/cookie/-/cookie-0.4.1.tgz#bfd02c1f2224567676c1545199f87c3a861d878d" @@ -1333,7 +1306,7 @@ dependencies: "@types/node" "*" -"@types/lodash@*", "@types/lodash@^4.14.182": +"@types/lodash@^4.14.182": version "4.14.191" resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.191.tgz#09511e7f7cba275acd8b419ddac8da9a6a79e2fa" integrity sha512-BdZ5BCCvho3EIXw6wUCXHe7rS53AIDPLE+JzwgT+OsJk53oBfbSmZZ7CX4VaRoN78N+TJpFi9QPlfIVNmJYWxQ== @@ -1428,16 +1401,6 @@ resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.3.13.tgz#da4bfd73f49bd541d28920ab0e2bf0ee80f71c91" integrity sha512-21cFJr9z3g5dW8B0CVI9g2O9beqaThGQ6ZFBqHfwhzLDKUxaqTIy3vnfah/UPkfOiF2pLq+tGz+W8RyCskuslw== -"@types/sequelize@^4.28.13": - version "4.28.14" - resolved "https://registry.yarnpkg.com/@types/sequelize/-/sequelize-4.28.14.tgz#70302689ddef09f5d472b548a14e5a150e690aff" - integrity sha512-O8lTJ8YPVVaoY9xjduchDlo0MOS3w262pro2H1QMuFIo/kc/p1elP/UxLOTP2wcVO2cFd6Gvghg9ZSAiJi0GLA== - dependencies: - "@types/bluebird" "*" - "@types/continuation-local-storage" "*" - "@types/lodash" "*" - "@types/validator" "*" - "@types/serve-static@*": version "1.15.0" resolved "https://registry.yarnpkg.com/@types/serve-static/-/serve-static-1.15.0.tgz#c7930ff61afb334e121a9da780aac0d9b8f34155" @@ -1451,7 +1414,7 @@ resolved "https://registry.yarnpkg.com/@types/tough-cookie/-/tough-cookie-4.0.2.tgz#6286b4c7228d58ab7866d19716f3696e03a09397" integrity sha512-Q5vtl1W5ue16D+nIaW8JWebSSraJVlK+EthKn7e7UcD4KWsaSJ8BqGPXNaPghgtcn/fhvrN17Tv8ksUsQpiplw== -"@types/validator@*", "@types/validator@^13.0.0", "@types/validator@^13.7.1", "@types/validator@^13.7.2": +"@types/validator@^13.0.0", "@types/validator@^13.7.1", "@types/validator@^13.7.2": version "13.7.10" resolved "https://registry.yarnpkg.com/@types/validator/-/validator-13.7.10.tgz#f9763dc0933f8324920afa9c0790308eedf55ca7" integrity sha512-t1yxFAR2n0+VO6hd/FJ9F2uezAZVWHLmpmlJzm1eX03+H7+HsuTAp7L8QJs+2pQCfWkP1+EXsGK9Z9v7o/qPVQ== @@ -1824,7 +1787,7 @@ bullmq@^1.87.0: tslib "^2.0.0" uuid "^9.0.0" -busboy@^1.0.0, busboy@^1.6.0: +busboy@^1.0.0: version "1.6.0" resolved "https://registry.yarnpkg.com/busboy/-/busboy-1.6.0.tgz#966ea36a9502e43cdb9146962523b92f531f6893" integrity sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA== @@ -2039,6 +2002,13 @@ cron-parser@^4.6.0: dependencies: luxon "^3.1.0" +cross-fetch@^3.1.6: + version "3.1.8" + resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-3.1.8.tgz#0327eba65fd68a7d119f8fb2bf9334a1a7956f82" + integrity sha512-cvA+JwZoU0Xq+h6WkMvAUqPEYy92Obet6UdKLfW60qn99ftItKjB5T+BkyWOFWe2pUyfQ+IJHmpOTznqk1M6Kg== + dependencies: + node-fetch "^2.6.12" + cross-spawn@^7.0.2, cross-spawn@^7.0.3: version "7.0.3" resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6" @@ -3496,6 +3466,13 @@ media-typer@0.3.0: resolved "https://registry.yarnpkg.com/media-typer/-/media-typer-0.3.0.tgz#8710d7af0aa626f8fffa1ce00168545263255748" integrity sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ== +meilisearch@^0.35.0: + version "0.35.0" + resolved "https://registry.yarnpkg.com/meilisearch/-/meilisearch-0.35.0.tgz#c94d5f266e39ad5ed5a666782e729805a4e8a956" + integrity sha512-gF1I6K5/Wpe7BWfjBnG+o19y/FIpJ9HbN+byON6CB9U3uE7qc6GvwUbjKOllh7LKXQVVxH/kCu7Jn0ODCUwqbQ== + dependencies: + cross-fetch "^3.1.6" + memoizee@^0.4.14: version "0.4.15" resolved "https://registry.yarnpkg.com/memoizee/-/memoizee-0.4.15.tgz#e6f3d2da863f318d02225391829a6c5956555b72" @@ -3634,7 +3611,7 @@ ms@2.1.2: resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== -ms@2.1.3, ms@^2.1.1, ms@^2.1.3: +ms@2.1.3, ms@^2.1.1: version "2.1.3" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== @@ -3702,6 +3679,13 @@ next-tick@1, next-tick@^1.1.0: resolved "https://registry.yarnpkg.com/next-tick/-/next-tick-1.1.0.tgz#1836ee30ad56d67ef281b22bd199f709449b35eb" integrity sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ== +node-fetch@^2.6.12: + version "2.7.0" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" + integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A== + dependencies: + whatwg-url "^5.0.0" + node-gyp-build-optional-packages@5.0.3: version "5.0.3" resolved "https://registry.yarnpkg.com/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.3.tgz#92a89d400352c44ad3975010368072b41ad66c17" @@ -4686,6 +4670,11 @@ tough-cookie@~2.5.0: psl "^1.1.28" punycode "^2.1.1" +tr46@~0.0.3: + version "0.0.3" + resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" + integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw== + triple-beam@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/triple-beam/-/triple-beam-1.3.0.tgz#a595214c7298db8339eeeee083e4d10bd8cb8dd9" @@ -4787,13 +4776,6 @@ unbox-primitive@^1.0.2: has-symbols "^1.0.3" which-boxed-primitive "^1.0.2" -undici@^5.5.1: - version "5.14.0" - resolved "https://registry.yarnpkg.com/undici/-/undici-5.14.0.tgz#1169d0cdee06a4ffdd30810f6228d57998884d00" - integrity sha512-yJlHYw6yXPPsuOH0x2Ib1Km61vu4hLiRRQoafs+WUgX1vO64vgnxiCEN9dpIrhZyHFsai3F0AEj4P9zy19enEQ== - dependencies: - busboy "^1.6.0" - universalify@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/universalify/-/universalify-2.0.0.tgz#75a4984efedc4b08975c5aeb73f530d02df25717" @@ -4855,6 +4837,19 @@ verror@1.10.0: core-util-is "1.0.2" extsprintf "^1.2.0" +webidl-conversions@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871" + integrity sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ== + +whatwg-url@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d" + integrity sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw== + dependencies: + tr46 "~0.0.3" + webidl-conversions "^3.0.0" + which-boxed-primitive@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz#13757bc89b209b049fe5d86430e21cf40a89a8e6"