import { CONFIG } from '../initializers/constants' import { DBVideo, DBVideoDetails, IndexableVideo, IndexableVideoDetails } from '../types/video.model' import { flatMap } from 'lodash' import { Avatar } from '@shared/models' import { buildSort, elasticSearch } from '../helpers/elastic-search' import { VideosSearchQuery } from '../types/video-search.model' import { inspect } from 'util' function initVideosIndex () { return elasticSearch.indices.create({ index: CONFIG.ELASTIC_SEARCH.INDEXES.VIDEOS, body: { settings: { number_of_shards: 1, number_of_replicas: 1 }, mappings: { properties: buildVideosMapping() } } }).catch(err => { if (err.name === 'ResponseError' && err.meta?.body?.error.root_cause[0]?.type === 'resource_already_exists_exception') return throw err }) } async function indexVideos (videos: IndexableVideo[]) { const elIdIndex: { [elId: string]: string } = {} for (const video of videos) { elIdIndex[video.elasticSearchId] = video.uuid } const body = flatMap(videos, v => { return [ { update: { _id: v.elasticSearchId, _index: CONFIG.ELASTIC_SEARCH.INDEXES.VIDEOS } }, { doc: formatVideo(v), doc_as_upsert: true } ] }) const result = await elasticSearch.bulk({ index: CONFIG.ELASTIC_SEARCH.INDEXES.VIDEOS, body }) const created: string[] = result.body.items .map(i => i.update) .filter(i => i.result === 'created') .map(i => elIdIndex[i._id]) return { created } } function refreshVideosIndex () { return elasticSearch.indices.refresh({ index: CONFIG.ELASTIC_SEARCH.INDEXES.VIDEOS }) } async function listIndexInstances () { const res = await elasticSearch.search({ index: CONFIG.ELASTIC_SEARCH.INDEXES.VIDEOS, body: { size: 0, aggs: { hosts: { terms: { field: 'host' } } } } }) return res.body.aggregations.hosts.buckets.map(b => b.key) } async function queryVideos (search: VideosSearchQuery) { const bool: any = {} const filter: any[] = [] if (search.search) { Object.assign(bool, { must: [ { multi_match: { query: search.search, fields: [ 'name', 'description' ] } } ] }) } if (search.startDate) { filter.push({ range: { publishedAt: { gte: search.startDate } } }) } if (search.endDate) { filter.push({ range: { publishedAt: { lte: search.endDate } } }) } if (search.originallyPublishedStartDate) { filter.push({ range: { originallyPublishedAt: { gte: search.startDate } } }) } if (search.originallyPublishedEndDate) { filter.push({ range: { originallyPublishedAt: { lte: search.endDate } } }) } if (search.nsfw) { filter.push({ term: { nsfw: search.nsfw } }) } if (search.categoryOneOf) { filter.push({ terms: { category: search.categoryOneOf } }) } if (search.licenceOneOf) { filter.push({ terms: { licence: search.licenceOneOf } }) } if (search.languageOneOf) { filter.push({ terms: { language: search.languageOneOf } }) } // FIXME: rework if (search.tagsOneOf) { filter.push({ terms: { tags: search.tagsOneOf, minimum_should_match: 1 } }) } if (search.tagsAllOf) { filter.push({ terms: { tags: search.tagsOneOf, minimum_should_match: 1 } }) } if (search.durationMin) { filter.push({ range: { duration: { gte: search.durationMin } } }) } if (search.durationMax) { filter.push({ range: { duration: { lte: search.durationMax } } }) } Object.assign(bool, { filter }) const res = await elasticSearch.search({ index: CONFIG.ELASTIC_SEARCH.INDEXES.VIDEOS, body: { from: search.start, size: search.count, sort: buildVideosSort(search.sort), query: { bool } } }) const hits = res.body.hits return { total: hits.total.value, data: hits.hits.map(h => h._source) } } export { indexVideos, queryVideos, refreshVideosIndex, initVideosIndex, listIndexInstances } // ############################################################################ function buildVideosSort (sort: string) { const { direction, field: sortField } = buildSort(sort) const field = sortField === 'match' ? '_score' : sortField return [ { [field]: { order: direction } } ] } function formatVideo (v: IndexableVideo | IndexableVideoDetails): DBVideo | DBVideoDetails { return { id: v.id, uuid: v.uuid, indexedAt: new Date(), createdAt: v.createdAt, updatedAt: v.updatedAt, publishedAt: v.publishedAt, originallyPublishedAt: v.originallyPublishedAt, category: { id: v.category.id, label: v.category.label }, licence: { id: v.licence.id, label: v.licence.label }, language: { id: v.language.id, label: v.language.label }, privacy: { id: v.privacy.id, label: v.privacy.label }, name: v.name, description: v.description, duration: v.duration, thumbnailPath: v.thumbnailPath, previewPath: v.previewPath, embedPath: v.embedPath, views: v.views, likes: v.likes, dislikes: v.dislikes, nsfw: v.nsfw, host: v.host, tags: (v as any).tags ? (v as any).tags : [], account: { id: v.account.id, name: v.account.name, displayName: v.account.displayName, url: v.account.url, host: v.account.host, avatar: formatAvatar(v.account) }, channel: { id: v.channel.id, name: v.channel.name, displayName: v.channel.displayName, url: v.channel.url, host: v.channel.host, avatar: formatAvatar(v.channel) } } } function formatAvatar (obj: { avatar?: Avatar }) { if (!obj.avatar) return null return { path: obj.avatar.path, createdAt: obj.avatar.createdAt, updatedAt: obj.avatar.updatedAt } } function buildChannelOrAccountMapping () { return { id: { type: 'long' }, name: { type: 'text', fields: { raw: { type: 'keyword' } } }, displayName: { type: 'text' }, url: { type: 'keyword' }, host: { type: 'keyword' }, avatar: { properties: { path: { type: 'keyword' }, createdAt: { type: 'date' }, updatedAt: { type: 'date' } } } } } function buildVideosMapping () { return { id: { type: 'long' }, uuid: { type: 'keyword' }, createdAt: { type: 'date' }, updatedAt: { type: 'date' }, publishedAt: { type: 'date' }, originallyPublishedAt: { type: 'date' }, indexedAt: { type: 'date' }, category: { properties: { id: { type: 'keyword' }, label: { type: 'text' } } }, licence: { properties: { id: { type: 'keyword' }, label: { type: 'text' } } }, language: { properties: { id: { type: 'keyword' }, label: { type: 'text' } } }, privacy: { properties: { id: { type: 'keyword' }, label: { type: 'text' } } }, name: { type: 'text' }, description: { type: 'text' }, tags: { type: 'text', fields: { raw: { type: 'keyword' } } }, duration: { type: 'long' }, thumbnailPath: { type: 'keyword' }, previewPath: { type: 'keyword' }, embedPath: { type: 'keyword' }, views: { type: 'long' }, likes: { type: 'long' }, dislikes: { type: 'long' }, nsfw: { type: 'boolean' }, host: { type: 'keyword' }, account: { properties: buildChannelOrAccountMapping() }, channel: { properties: buildChannelOrAccountMapping() } } }