sepia-search-motore-di-ricerca-video/server/lib/elastic-search-videos.ts

import { CONFIG } from '../initializers/constants'
import { DBVideo, DBVideoDetails, IndexableVideo, IndexableVideoDetails } from '../types/video.model'
import { flatMap } from 'lodash'
import { Avatar, Video } from '@shared/models'
import { buildSort, elasticSearch } from '../helpers/elastic-search'
import { VideosSearchQuery } from '../types/video-search.model'
import { logger } from '../helpers/logger'

function initVideosIndex () {
  logger.info('Initialize %s Elastic Search index.', CONFIG.ELASTIC_SEARCH.INDEXES.VIDEOS)

  return elasticSearch.indices.create({
    index: CONFIG.ELASTIC_SEARCH.INDEXES.VIDEOS,
    body: {
      settings: {
        number_of_shards: 1,
        number_of_replicas: 1
      },
      mappings: {
        properties: buildVideosMapping()
      }
    }
  }).catch(err => {
    if (err.name === 'ResponseError' && err.meta?.body?.error.root_cause[0]?.type === 'resource_already_exists_exception') return

    throw err
  })
}

async function indexVideos (videos: IndexableVideo[], replace = false) {
  const elIdIndex: { [elId: string]: string } = {}

  for (const video of videos) {
    elIdIndex[video.elasticSearchId] = video.uuid
  }

  const method = replace ? 'index' : 'update'

  const body = flatMap(videos, v => {
    const doc = formatVideoForDB(v)

    const options = replace
      ? doc
      : { doc, doc_as_upsert: true }

    return [
      {
        [method]: {
          _id: v.elasticSearchId,
          _index: CONFIG.ELASTIC_SEARCH.INDEXES.VIDEOS
        }
      },
      options
    ]
  })

  const result = await elasticSearch.bulk({
    index: CONFIG.ELASTIC_SEARCH.INDEXES.VIDEOS,
    body
  })

  const resultBody = result.body

  if (resultBody.errors === true) {
    const msg = 'Cannot insert data in elastic search.'
    logger.error(msg, { err: resultBody })
    throw new Error(msg)
  }

  const created: string[] = result.body.items
                                  .map(i => i[method])
                                  .filter(i => i.result === 'created')
                                  .map(i => elIdIndex[i._id])

  return { created }
}

function refreshVideosIndex () {
  return elasticSearch.indices.refresh({ index: CONFIG.ELASTIC_SEARCH.INDEXES.VIDEOS })
}

async function listIndexInstances () {
  const res = await elasticSearch.search({
    index: CONFIG.ELASTIC_SEARCH.INDEXES.VIDEOS,
    body: {
      size: 0,
      aggs: {
        hosts: {
          terms: {
            field: 'host'
          }
        }
      }
    }
  })

  return res.body.aggregations.hosts.buckets.map(b => b.key)
}

async function queryVideos (search: VideosSearchQuery) {
  const bool: any = {}
  const filter: any[] = []

  if (search.search) {
    Object.assign(bool, {
      must: [
        {
          multi_match: {
            query: search.search,
            fields: [ 'name', 'description', 'tags' ],
            fuzziness: 'AUTO'
          }
        }
      ]
    })
  }

  if (search.startDate) {
    filter.push({
      range: {
        publishedAt: {
          gte: search.startDate
        }
      }
    })
  }

  if (search.endDate) {
    filter.push({
      range: {
        publishedAt: {
          lte: search.endDate
        }
      }
    })
  }

  if (search.originallyPublishedStartDate) {
    filter.push({
      range: {
        originallyPublishedAt: {
          gte: search.startDate
        }
      }
    })
  }

  if (search.originallyPublishedEndDate) {
    filter.push({
      range: {
        originallyPublishedAt: {
          lte: search.endDate
        }
      }
    })
  }

  if (search.nsfw && search.nsfw !== 'both') {
    filter.push({
      term: {
        nsfw: (search.nsfw + '') === 'true'
      }
    })
  }

  if (search.categoryOneOf) {
    filter.push({
      terms: {
        'category.id': search.categoryOneOf
      }
    })
  }

  if (search.licenceOneOf) {
    filter.push({
      terms: {
        'licence.id': search.licenceOneOf
      }
    })
  }

  if (search.languageOneOf) {
    filter.push({
      terms: {
        'language.id': search.languageOneOf
      }
    })
  }

  if (search.tagsOneOf) {
    filter.push({
      terms: {
        tags: search.tagsOneOf
      }
    })
  }

  if (search.tagsAllOf) {
    for (const t of search.tagsAllOf) {
      filter.push({
        term: {
          tags: t
        }
      })
    }
  }

  if (search.durationMin) {
    filter.push({
      range: {
        duration: {
          gte: search.durationMin
        }
      }
    })
  }

  if (search.durationMax) {
    filter.push({
      range: {
        duration: {
          lte: search.durationMax
        }
      }
    })
  }

  Object.assign(bool, { filter })

  const body = {
    from: search.start,
    size: search.count,
    sort: buildVideosSort(search.sort),
    query: { bool }
  }

  logger.debug('Will query Elastic Search for videos.', { body })

  const res = await elasticSearch.search({
    index: CONFIG.ELASTIC_SEARCH.INDEXES.VIDEOS,
    body
  })

  const hits = res.body.hits
  return { total: hits.total.value, data: hits.hits.map(h => h._source) }
}

export {
  indexVideos,
  queryVideos,
  refreshVideosIndex,
  initVideosIndex,
  listIndexInstances,
  formatVideoForAPI
}

// ############################################################################

function buildVideosSort (sort: string) {
  const { direction, field: sortField } = buildSort(sort)

  const field = sortField === 'match'
    ? '_score'
    : sortField

  return [
    {
      [field]: { order: direction }
    }
  ]
}

function formatVideoForDB (v: IndexableVideo | IndexableVideoDetails): DBVideo | DBVideoDetails {
  return {
    id: v.id,
    uuid: v.uuid,

    indexedAt: new Date(),
    createdAt: v.createdAt,
    updatedAt: v.updatedAt,
    publishedAt: v.publishedAt,
    originallyPublishedAt: v.originallyPublishedAt,

    category: {
      id: v.category.id,
      label: v.category.label
    },
    licence: {
      id: v.licence.id,
      label: v.licence.label
    },
    language: {
      id: v.language.id,
      label: v.language.label
    },
    privacy: {
      id: v.privacy.id,
      label: v.privacy.label
    },

    name: v.name,
    description: v.description,
    duration: v.duration,
    thumbnailPath: v.thumbnailPath,
    previewPath: v.previewPath,
    embedPath: v.embedPath,

    views: v.views,
    likes: v.likes,
    dislikes: v.dislikes,

    nsfw: v.nsfw,
    host: v.host,

    tags: (v as IndexableVideoDetails).tags ? (v as IndexableVideoDetails).tags : undefined,

    account: {
      id: v.account.id,
      name: v.account.name,
      displayName: v.account.displayName,
      url: v.account.url,
      host: v.account.host,

      avatar: formatAvatarForDB(v.account)
    },

    channel: {
      id: v.channel.id,
      name: v.channel.name,
      displayName: v.channel.displayName,
      url: v.channel.url,
      host: v.channel.host,

      avatar: formatAvatarForDB(v.channel)
    }
  }
}

function formatAvatarForDB (obj: { avatar?: Avatar }) {
  if (!obj.avatar) return null

  return {
    path: obj.avatar.path,
    createdAt: obj.avatar.createdAt,
    updatedAt: obj.avatar.updatedAt
  }
}

function formatVideoForAPI (v: DBVideo, fromHost?: string): Video {
  return {
    id: v.id,
    uuid: v.uuid,

    createdAt: new Date(v.createdAt),
    updatedAt: new Date(v.updatedAt),
    publishedAt: new Date(v.publishedAt),
    originallyPublishedAt: v.originallyPublishedAt,

    category: {
      id: v.category.id,
      label: v.category.label
    },
    licence: {
      id: v.licence.id,
      label: v.licence.label
    },
    language: {
      id: v.language.id,
      label: v.language.label
    },
    privacy: {
      id: v.privacy.id,
      label: v.privacy.label
    },

    name: v.name,
    description: v.description,
    duration: v.duration,
    thumbnailPath: v.thumbnailPath,
    previewPath: v.previewPath,
    embedPath: v.embedPath,
    isLocal: fromHost && fromHost === v.host,

    views: v.views,
    likes: v.likes,
    dislikes: v.dislikes,

    nsfw: v.nsfw,

    account: {
      id: v.account.id,
      name: v.account.name,
      displayName: v.account.displayName,
      url: v.account.url,
      host: v.account.host,

      avatar: formatAvatarForAPI(v.account)
    },

    channel: {
      id: v.channel.id,
      name: v.channel.name,
      displayName: v.channel.displayName,
      url: v.channel.url,
      host: v.channel.host,

      avatar: formatAvatarForAPI(v.channel)
    }
  }
}

function formatAvatarForAPI (obj: { avatar?: Avatar }) {
  if (!obj.avatar) return null

  return {
    path: obj.avatar.path,
    createdAt: obj.avatar.createdAt,
    updatedAt: obj.avatar.updatedAt
  }
}

function buildChannelOrAccountMapping () {
  return {
    id: {
      type: 'long'
    },

    name: {
      type: 'text',
      fields: {
        raw: {
          type: 'keyword'
        }
      }
    },
    displayName: {
      type: 'text'
    },
    url: {
      type: 'keyword'
    },
    host: {
      type: 'keyword'
    },

    avatar: {
      properties: {
        path: {
          type: 'keyword'
        },
        createdAt: {
          type: 'date',
          format: 'date_optional_time'
        },
        updatedAt: {
          type: 'date',
          format: 'date_optional_time'
        }
      }
    }
  }
}

function buildVideosMapping () {
  return {
    id: {
      type: 'long'
    },

    uuid: {
      type: 'keyword'
    },
    createdAt: {
      type: 'date',
      format: 'date_optional_time'
    },
    updatedAt: {
      type: 'date',
      format: 'date_optional_time'
    },
    publishedAt: {
      type: 'date',
      format: 'date_optional_time'
    },
    originallyPublishedAt: {
      type: 'date',
      format: 'date_optional_time'
    },
    indexedAt: {
      type: 'date',
      format: 'date_optional_time'
    },

    category: {
      properties: {
        id: {
          type: 'keyword'
        },
        label: {
          type: 'text'
        }
      }
    },

    licence: {
      properties: {
        id: {
          type: 'keyword'
        },
        label: {
          type: 'text'
        }
      }
    },

    language: {
      properties: {
        id: {
          type: 'keyword'
        },
        label: {
          type: 'text'
        }
      }
    },

    privacy: {
      properties: {
        id: {
          type: 'keyword'
        },
        label: {
          type: 'text'
        }
      }
    },

    name: {
      type: 'text'
    },

    description: {
      type: 'text'
    },

    tags: {
      type: 'text',

      fields: {
        raw: {
          type: 'keyword'
        }
      }
    },

    duration: {
      type: 'long'
    },

    thumbnailPath: {
      type: 'keyword'
    },
    previewPath: {
      type: 'keyword'
    },
    embedPath: {
      type: 'keyword'
    },

    views: {
      type: 'long'
    },
    likes: {
      type: 'long'
    },
    dislikes: {
      type: 'long'
    },
    nsfw: {
      type: 'boolean'
    },

    host: {
      type: 'keyword'
    },

    account: {
      properties: buildChannelOrAccountMapping()
    },

    channel: {
      properties: buildChannelOrAccountMapping()
    }
  }
}