import { difference } from 'lodash' import { buildIndex, buildSort, elasticSearch, extractQueryResult, indexDocuments } from '../helpers/elastic-search' import { logger } from '../helpers/logger' import { CONFIG, ELASTIC_SEARCH_QUERY } from '../initializers/constants' import { ChannelsSearchQuery } from '../types/channel-search.model' import { DBChannel, EnhancedVideoChannel, IndexableChannel } from '../types/channel.model' import { buildAvatarMapping, formatAvatarForAPI, formatAvatarForDB } from './elastic-search-avatar' function initChannelsIndex () { return buildIndex(CONFIG.ELASTIC_SEARCH.INDEXES.CHANNELS, buildChannelsMapping()) } async function indexChannels (channels: IndexableChannel[], replace = false) { return indexDocuments({ objects: channels, formatter: c => formatChannelForDB(c), replace, index: CONFIG.ELASTIC_SEARCH.INDEXES.CHANNELS }) } function refreshChannelsIndex () { logger.info('Refreshing channels index.') return elasticSearch.indices.refresh({ index: CONFIG.ELASTIC_SEARCH.INDEXES.CHANNELS }) } async function removeNotExistingChannels (host: string, existingChannels: Set) { const idsFromDB = await getChannelIdsOf(host) const idsToRemove = difference(idsFromDB, Array.from(existingChannels)) logger.info({ idsToRemove }, 'Will remove %d channels from %s.', idsToRemove.length, host) return elasticSearch.delete_by_query({ index: CONFIG.ELASTIC_SEARCH.INDEXES.CHANNELS, body: { query: { bool: { filter: [ { terms: { id: idsToRemove } }, { term: { host } } ] } } } }) } function removeChannelsFromHosts (hosts: string[]) { if (hosts.length === 0) return logger.info({ hosts }, 'Will remove channels from hosts.') return elasticSearch.delete_by_query({ index: CONFIG.ELASTIC_SEARCH.INDEXES.CHANNELS, body: { query: { bool: { filter: { terms: { host: hosts } } } } } }) } async function queryChannels (search: ChannelsSearchQuery) { const bool: any = {} const mustNot: any[] = [] if (search.search) { Object.assign(bool, { must: [ { multi_match: { query: search.search, fields: ELASTIC_SEARCH_QUERY.CHANNELS_MULTI_MATCH_FIELDS, fuzziness: ELASTIC_SEARCH_QUERY.FUZZINESS } } ] }) } if (search.blockedAccounts) { mustNot.push({ terms: { 'ownerAccount.handle': search.blockedAccounts } }) } if (search.blockedHosts) { mustNot.push({ terms: { host: search.blockedHosts } }) } if (mustNot.length !== 0) { Object.assign(bool, { must_not: mustNot }) } const body = { from: search.start, size: search.count, sort: buildSort(search.sort), query: { bool } } logger.debug({ body }, 'Will query Elastic Search for channels.') const res = await elasticSearch.search({ index: CONFIG.ELASTIC_SEARCH.INDEXES.CHANNELS, body }) return extractQueryResult(res) } async function getChannelIdsOf (host: string) { const res = await elasticSearch.search({ index: CONFIG.ELASTIC_SEARCH.INDEXES.CHANNELS, body: { size: 0, aggs: { ids: { terms: { field: 'id' } } }, query: { bool: { filter: [ { term: { host } } ] } } } }) return res.body.aggregations.ids.buckets.map(b => b.key) } export { removeChannelsFromHosts, initChannelsIndex, indexChannels, refreshChannelsIndex, formatChannelForAPI, queryChannels, getChannelIdsOf, removeNotExistingChannels } // ############################################################################ function formatChannelForDB (c: IndexableChannel): DBChannel { return { id: c.id, name: c.name, host: c.host, url: c.url, avatar: formatAvatarForDB(c), displayName: c.displayName, indexedAt: new Date(), followingCount: c.followingCount, followersCount: c.followersCount, createdAt: c.createdAt, updatedAt: c.updatedAt, description: c.description, support: c.support, handle: `${c.name}@${c.host}`, ownerAccount: { id: c.ownerAccount.id, url: c.ownerAccount.url, displayName: c.ownerAccount.displayName, description: c.ownerAccount.description, name: c.ownerAccount.name, host: c.ownerAccount.host, followingCount: c.ownerAccount.followingCount, followersCount: c.ownerAccount.followersCount, createdAt: c.ownerAccount.createdAt, updatedAt: c.ownerAccount.updatedAt, handle: `${c.ownerAccount.name}@${c.ownerAccount.host}`, avatar: formatAvatarForDB(c.ownerAccount) } } } function formatChannelForAPI (c: DBChannel, fromHost?: string): EnhancedVideoChannel { return { id: c.id, score: c.score, url: c.url, name: c.name, host: c.host, followingCount: c.followingCount, followersCount: c.followersCount, createdAt: c.createdAt, updatedAt: c.updatedAt, avatar: formatAvatarForAPI(c), displayName: c.displayName, description: c.description, support: c.support, isLocal: fromHost === c.host, ownerAccount: { id: c.ownerAccount.id, url: c.ownerAccount.url, displayName: c.ownerAccount.displayName, description: c.ownerAccount.description, name: c.ownerAccount.name, host: c.ownerAccount.host, followingCount: c.ownerAccount.followingCount, followersCount: c.ownerAccount.followersCount, createdAt: c.ownerAccount.createdAt, updatedAt: c.ownerAccount.updatedAt, avatar: formatAvatarForAPI(c.ownerAccount) } } } function buildChannelOrAccountCommonMapping () { return { id: { type: 'long' }, url: { type: 'keyword' }, name: { type: 'text', fields: { raw: { type: 'keyword' } } }, host: { type: 'keyword' }, handle: { type: 'keyword' }, displayName: { type: 'text' }, avatar: { properties: buildAvatarMapping() }, followingCount: { type: 'long' }, followersCount: { type: 'long' }, createdAt: { type: 'date', format: 'date_optional_time' }, updatedAt: { type: 'date', format: 'date_optional_time' }, description: { type: 'text' } } } function buildChannelsMapping () { const base = buildChannelOrAccountCommonMapping() Object.assign(base, { support: { type: 'keyword' }, ownerAccount: { properties: buildChannelOrAccountCommonMapping() } }) return base }