sepia-search-motore-di-ricerca-video/server/lib/elastic-search-channels.ts

import { difference } from 'lodash'
import { buildIndex, buildSort, elasticSearch, extractQueryResult, indexDocuments } from '../helpers/elastic-search'
import { logger } from '../helpers/logger'
import { CONFIG, ELASTIC_SEARCH_QUERY } from '../initializers/constants'
import { ChannelsSearchQuery } from '../types/channel-search.model'
import { DBChannel, EnhancedVideoChannel, IndexableChannel } from '../types/channel.model'
import { buildAvatarMapping, formatAvatarForAPI, formatAvatarForDB } from './elastic-search-avatar'

function initChannelsIndex () {
  return buildIndex(CONFIG.ELASTIC_SEARCH.INDEXES.CHANNELS, buildChannelsMapping())
}

async function indexChannels (channels: IndexableChannel[], replace = false) {
  return indexDocuments({
    objects: channels,
    formatter: c => formatChannelForDB(c),
    replace,
    index: CONFIG.ELASTIC_SEARCH.INDEXES.CHANNELS
  })
}

function refreshChannelsIndex () {
  logger.info('Refreshing channels index.')

  return elasticSearch.indices.refresh({ index: CONFIG.ELASTIC_SEARCH.INDEXES.CHANNELS })
}

async function removeNotExistingChannels (host: string, existingChannels: Set<number>) {
  const idsFromDB = await getChannelIdsOf(host)

  const idsToRemove = difference(idsFromDB, Array.from(existingChannels))

  logger.info({ idsToRemove }, 'Will remove %d channels from %s.', idsToRemove.length, host)

  return elasticSearch.delete_by_query({
    index: CONFIG.ELASTIC_SEARCH.INDEXES.CHANNELS,
    body: {
      query: {
        bool: {
          filter: [
            {
              terms: {
                id: idsToRemove
              }
            },
            {
              term: {
                host
              }
            }
          ]
        }
      }
    }
  })
}

function removeChannelsFromHosts (hosts: string[]) {
  if (hosts.length === 0) return

  logger.info({ hosts }, 'Will remove channels from hosts.')

  return elasticSearch.delete_by_query({
    index: CONFIG.ELASTIC_SEARCH.INDEXES.CHANNELS,
    body: {
      query: {
        bool: {
          filter: {
            terms: {
              host: hosts
            }
          }
        }
      }
    }
  })
}

async function queryChannels (search: ChannelsSearchQuery) {
  const bool: any = {}
  const mustNot: any[] = []

  if (search.search) {
    Object.assign(bool, {
      must: [
        {
          multi_match: {
            query: search.search,
            fields: ELASTIC_SEARCH_QUERY.CHANNELS_MULTI_MATCH_FIELDS,
            fuzziness: ELASTIC_SEARCH_QUERY.FUZZINESS
          }
        }
      ]
    })
  }

  if (search.blockedAccounts) {
    mustNot.push({
      terms: {
        'ownerAccount.handle': search.blockedAccounts
      }
    })
  }

  if (search.blockedHosts) {
    mustNot.push({
      terms: {
        host: search.blockedHosts
      }
    })
  }

  if (mustNot.length !== 0) {
    Object.assign(bool, { must_not: mustNot })
  }

  const body = {
    from: search.start,
    size: search.count,
    sort: buildSort(search.sort),
    query: { bool }
  }

  logger.debug({ body }, 'Will query Elastic Search for channels.')

  const res = await elasticSearch.search({
    index: CONFIG.ELASTIC_SEARCH.INDEXES.CHANNELS,
    body
  })

  return extractQueryResult(res)
}

async function getChannelIdsOf (host: string) {
  const res = await elasticSearch.search({
    index: CONFIG.ELASTIC_SEARCH.INDEXES.CHANNELS,
    body: {
      size: 0,
      aggs: {
        ids: {
          terms: {
            field: 'id'
          }
        }
      },
      query: {
        bool: {
          filter: [
            {
              term: {
                host
              }
            }
          ]
        }
      }
    }
  })

  return res.body.aggregations.ids.buckets.map(b => b.key)
}

export {
  removeChannelsFromHosts,
  initChannelsIndex,
  indexChannels,
  refreshChannelsIndex,
  formatChannelForAPI,
  queryChannels,
  getChannelIdsOf,
  removeNotExistingChannels
}

// ############################################################################

function formatChannelForDB (c: IndexableChannel): DBChannel {
  return {
    id: c.id,

    name: c.name,
    host: c.host,
    url: c.url,

    avatar: formatAvatarForDB(c),

    displayName: c.displayName,

    indexedAt: new Date(),

    followingCount: c.followingCount,
    followersCount: c.followersCount,
    createdAt: c.createdAt,
    updatedAt: c.updatedAt,

    description: c.description,
    support: c.support,

    handle: `${c.name}@${c.host}`,

    ownerAccount: {
      id: c.ownerAccount.id,
      url: c.ownerAccount.url,

      displayName: c.ownerAccount.displayName,
      description: c.ownerAccount.description,
      name: c.ownerAccount.name,
      host: c.ownerAccount.host,
      followingCount: c.ownerAccount.followingCount,
      followersCount: c.ownerAccount.followersCount,
      createdAt: c.ownerAccount.createdAt,
      updatedAt: c.ownerAccount.updatedAt,

      handle: `${c.ownerAccount.name}@${c.ownerAccount.host}`,

      avatar: formatAvatarForDB(c.ownerAccount)
    }
  }
}

function formatChannelForAPI (c: DBChannel, fromHost?: string): EnhancedVideoChannel {
  return {
    id: c.id,

    score: c.score,

    url: c.url,
    name: c.name,
    host: c.host,
    followingCount: c.followingCount,
    followersCount: c.followersCount,
    createdAt: c.createdAt,
    updatedAt: c.updatedAt,
    avatar: formatAvatarForAPI(c),

    displayName: c.displayName,
    description: c.description,
    support: c.support,
    isLocal: fromHost === c.host,

    ownerAccount: {
      id: c.ownerAccount.id,
      url: c.ownerAccount.url,

      displayName: c.ownerAccount.displayName,
      description: c.ownerAccount.description,
      name: c.ownerAccount.name,
      host: c.ownerAccount.host,
      followingCount: c.ownerAccount.followingCount,
      followersCount: c.ownerAccount.followersCount,
      createdAt: c.ownerAccount.createdAt,
      updatedAt: c.ownerAccount.updatedAt,

      avatar: formatAvatarForAPI(c.ownerAccount)
    }
  }
}

function buildChannelOrAccountCommonMapping () {
  return {
    id: {
      type: 'long'
    },

    url: {
      type: 'keyword'
    },

    name: {
      type: 'text',
      fields: {
        raw: {
          type: 'keyword'
        }
      }
    },

    host: {
      type: 'keyword'
    },

    handle: {
      type: 'keyword'
    },

    displayName: {
      type: 'text'
    },

    avatar: {
      properties: buildAvatarMapping()
    },

    followingCount: {
      type: 'long'
    },
    followersCount: {
      type: 'long'
    },

    createdAt: {
      type: 'date',
      format: 'date_optional_time'
    },
    updatedAt: {
      type: 'date',
      format: 'date_optional_time'
    },

    description: {
      type: 'text'
    }
  }
}

function buildChannelsMapping () {
  const base = buildChannelOrAccountCommonMapping()

  Object.assign(base, {
    support: {
      type: 'keyword'
    },

    ownerAccount: {
      properties: buildChannelOrAccountCommonMapping()
    }
  })

  return base
}