Upgrade server dependendencies

README.md

@@ -32,7 +32,7 @@ Add the locale in `client/src/main.ts` and `client/Makefile`. Then update transl
 ## Production
 
 Install dependencies:
-  * NodeJS (v12)
+  * NodeJS (v16)
   * Elastic Search
 
 ```terminal

package.json

@@ -22,52 +22,52 @@
     "i18n:update": "cd client && git fetch weblate && git merge weblate/master && npm run gettext:extract && npm run gettext:compile"
   },
   "dependencies": {
-    "@elastic/elasticsearch": "^7.6.0",
-    "async": "^3.1.0",
+    "@elastic/elasticsearch": "^8.2.1",
+    "async": "^3.2.3",
     "bluebird": "^3.5.3",
-    "body-parser": "^1.12.4",
-    "config": "^3.0.1",
+    "body-parser": "^1.20.0",
+    "config": "^3.3.7",
     "cors": "^2.8.5",
-    "express": "^4.16.4",
-    "express-validator": "^6.1.1",
-    "fs-extra": "^10.0.0",
+    "express": "^4.18.1",
+    "express-validator": "^6.14.1",
+    "fs-extra": "^10.1.0",
     "js-yaml": "^4.1.0",
     "lodash": "^4.17.15",
     "mkdirp": "^1.0.4",
     "morgan": "^1.9.1",
-    "multer": "^1.4.1",
-    "pg": "^8.2.1",
-    "pino": "^6.3.0",
-    "pino-pretty": "^5.1.0",
+    "multer": "^1.4.5-lts.1",
+    "pg": "^8.7.3",
+    "pino": "^8.0.0",
+    "pino-pretty": "^8.0.0",
     "reflect-metadata": "^0.1.13",
     "request": "^2.88.0",
     "retry": "^0.13.1",
-    "source-map-support": "^0.5.10"
+    "source-map-support": "^0.5.21"
   },
   "devDependencies": {
-    "@types/async": "^3.0.0",
-    "@types/body-parser": "^1.16.3",
-    "@types/config": "^0.0.38",
-    "@types/express": "^4.16.1",
-    "@types/fluent-ffmpeg": "^2.1.14",
-    "@types/fs-extra": "^9.0.1",
-    "@types/lodash": "^4.14.149",
-    "@types/mkdirp": "^1.0.0",
-    "@types/morgan": "^1.7.32",
-    "@types/multer": "^1.3.3",
-    "@types/node": "^15.12.4",
-    "@types/pino": "^6.0.1",
-    "@types/request": "^2.48.1",
-    "@types/sequelize": "^4.27.35",
-    "@types/validator": "^13.0.0",
-    "@typescript-eslint/eslint-plugin": "^4.28.0",
-    "@typescript-eslint/parser": "^4.28.0",
-    "eslint": "^7.1.0",
-    "eslint-config-standard-with-typescript": "^20.0.0",
-    "eslint-plugin-import": "^2.20.1",
+    "@types/async": "^3.2.13",
+    "@types/body-parser": "^1.19.2",
+    "@types/config": "^0.0.41",
+    "@types/express": "^4.17.13",
+    "@types/fluent-ffmpeg": "^2.1.20",
+    "@types/fs-extra": "^9.0.13",
+    "@types/lodash": "^4.14.182",
+    "@types/mkdirp": "^1.0.2",
+    "@types/morgan": "^1.9.3",
+    "@types/multer": "^1.4.7",
+    "@types/node": "^17.0.38",
+    "@types/pino": "^7.0.5",
+    "@types/request": "^2.48.8",
+    "@types/sequelize": "^4.28.13",
+    "@types/validator": "^13.7.2",
+    "@typescript-eslint/eslint-plugin": "^5.27.0",
+    "@typescript-eslint/parser": "^5.27.0",
+    "eslint": "^8.16.0",
+    "eslint-config-standard-with-typescript": "^21.0.1",
+    "eslint-plugin-import": "^2.26.0",
     "eslint-plugin-node": "^11.0.0",
-    "eslint-plugin-promise": "^5.1.0",
+    "eslint-plugin-promise": "^6.0.0",
     "eslint-plugin-standard": "^5.0.0",
-    "typescript": "^4.3.4"
+    "typescript": "^4.7.2"
   }
 }

server.ts

@@ -4,10 +4,10 @@ if (isTestInstance()) {
   require('source-map-support').install()
 }
 
-import * as bodyParser from 'body-parser'
-import * as express from 'express'
-import * as cors from 'cors'
-import * as morgan from 'morgan'
+import bodyParser from 'body-parser'
+import express from 'express'
+import cors from 'cors'
+import morgan from 'morgan'
 import { apiRouter } from './server/controllers/api'
 import { logger } from './server/helpers/logger'
 import { API_VERSION, CONFIG, getWebserverUrl } from './server/initializers/constants'

server/controllers/api/config.ts

@@ -1,4 +1,4 @@
-import * as express from 'express'
+import express from 'express'
 import { ServerConfig } from '../../../shared'
 import { CONFIG } from '../../initializers/constants'
 import { IndexationScheduler } from '../../lib/schedulers/indexation-scheduler'

server/controllers/api/index.ts

@@ -1,4 +1,4 @@
-import * as express from 'express'
+import express from 'express'
 import { badRequest } from '../../helpers/utils'
 import { configRouter } from './config'
 import { searchChannelsRouter } from './search-channels'

server/controllers/api/search-channels.ts

@@ -1,4 +1,4 @@
-import * as express from 'express'
+import express from 'express'
 import { Searcher } from '../../lib/controllers/searcher'
 import { formatChannelForAPI, queryChannels } from '../../lib/elastic-search/elastic-search-channels'
 import { asyncMiddleware } from '../../middlewares/async'

server/controllers/api/search-playlists.ts

@@ -1,4 +1,4 @@
-import * as express from 'express'
+import express from 'express'
 import { Searcher } from '../../lib/controllers/searcher'
 import { formatPlaylistForAPI, queryPlaylists } from '../../lib/elastic-search/elastic-search-playlists'
 import { asyncMiddleware } from '../../middlewares/async'

server/controllers/api/search-videos.ts

@@ -1,4 +1,4 @@
-import * as express from 'express'
+import express from 'express'
 import { Searcher } from '../../lib/controllers/searcher'
 import { formatVideoForAPI, queryVideos } from '../../lib/elastic-search/elastic-search-videos'
 import { asyncMiddleware } from '../../middlewares/async'

server/helpers/logger.ts

@@ -1,8 +1,10 @@
-import * as Pino from 'pino'
+import Pino from 'pino'
 import { CONFIG } from '../initializers/constants'
 
 const pino = Pino({
-  prettyPrint: true,
+  transport: {
+    target: 'pino-pretty'
+  },
   level: CONFIG.LOG.LEVEL
 })
 

server/helpers/requests.ts

@@ -1,5 +1,5 @@
-import * as Bluebird from 'bluebird'
-import * as request from 'request'
+import Bluebird from 'bluebird'
+import request from 'request'
 import { getWebserverUrl } from '../initializers/constants'
 import { waitMs } from './core-utils'
 

server/helpers/utils.ts

@@ -1,4 +1,4 @@
-import * as express from 'express'
+import express from 'express'
 import { ResultList } from '../../PeerTube/shared/models/common/result-list.model'
 
 function badRequest (req: express.Request, res: express.Response) {

server/initializers/constants.ts

@@ -1,4 +1,4 @@
-import * as config from 'config'
+import config from 'config'
 import { isTestInstance } from '../helpers/core-utils'
 
 const API_VERSION = 'v1'

server/lib/elastic-search/elastic-search-channels.ts

@@ -1,9 +1,10 @@
+import { MappingProperty, PropertyName } from '@elastic/elasticsearch/lib/api/types'
 import { elasticSearch } from '../../helpers/elastic-search'
 import { logger } from '../../helpers/logger'
 import { CONFIG, ELASTIC_SEARCH_QUERY } from '../../initializers/constants'
 import { DBChannel, EnhancedVideoChannel, IndexableChannel } from '../../types/channel.model'
 import { ChannelsSearchQuery } from '../../types/search-query/channel-search.model'
-import { buildSort, extractQueryResult } from './elastic-search-queries'
+import { buildSort, extractSearchQueryResult } from './elastic-search-queries'
 import { buildChannelOrAccountCommonMapping, buildMultiMatchBool } from './shared'
 import {
   formatActorImageForAPI,
@@ -75,7 +76,7 @@ async function queryChannels (search: ChannelsSearchQuery) {
     body
   })
 
-  return extractQueryResult(res)
+  return extractSearchQueryResult(res)
 }
 
 function formatChannelForAPI (c: DBChannel, fromHost?: string): EnhancedVideoChannel {
@@ -187,7 +188,7 @@ function buildChannelsMapping () {
     ownerAccount: {
       properties: buildChannelOrAccountCommonMapping()
     }
-  })
+  } as Record<PropertyName, MappingProperty>)
 
   return base
 }

server/lib/elastic-search/elastic-search-index.ts

@@ -1,9 +1,10 @@
 import { flatMap } from 'lodash'
+import { MappingProperty, PropertyName } from '@elastic/elasticsearch/lib/api/types'
 import { elasticSearch } from '../../helpers/elastic-search'
 import { logger } from '../../helpers/logger'
 import { IndexableDoc } from '../../types/indexable-doc.model'
 
-function buildIndex (name: string, mapping: object) {
+function buildIndex (name: string, mapping: Record<PropertyName, MappingProperty>) {
   logger.info('Initialize %s Elastic Search index.', name)
 
   return elasticSearch.indices.create({
@@ -63,18 +64,16 @@ async function indexDocuments <T extends IndexableDoc> (options: {
     body
   })
 
-  const resultBody = result.body
-
-  if (resultBody.errors === true) {
+  if (result.errors === true) {
     const msg = 'Cannot insert data in elastic search.'
-    logger.error({ err: resultBody }, msg)
+    logger.error({ err: result }, msg)
     throw new Error(msg)
   }
 
-  const created: T[] = result.body.items
-                                  .map(i => i[method])
-                                  .filter(i => i.result === 'created')
-                                  .map(i => elIdIndex[i._id])
+  const created: T[] = result.items
+                             .map(i => i[method])
+                             .filter(i => i.result === 'created')
+                             .map(i => elIdIndex[i._id])
 
   return { created }
 }

server/lib/elastic-search/elastic-search-instances.ts

@@ -1,6 +1,8 @@
+import { AggregationsStringTermsAggregate } from '@elastic/elasticsearch/lib/api/types'
 import { elasticSearch } from '../../helpers/elastic-search'
 import { CONFIG } from '../../initializers/constants'
 import { listIndexInstancesHost } from '../requests/instances-index'
+import { extractBucketsFromAggregation } from './elastic-search-queries'
 
 async function buildInstanceHosts () {
   let indexHosts = await listIndexInstancesHost()
@@ -33,7 +35,7 @@ async function listDBInstances () {
   ]
 
   for (const index of indexes) {
-    const res = await elasticSearch.search({
+    const res = await elasticSearch.search<unknown, Record<'hosts', AggregationsStringTermsAggregate>>({
       index,
       body: {
         size: 0,
@@ -48,7 +50,7 @@ async function listDBInstances () {
       }
     })
 
-    for (const b of res.body.aggregations.hosts.buckets) {
+    for (const b of extractBucketsFromAggregation<string>(res.aggregations.hosts.buckets)) {
       setResult.add(b.key)
     }
   }

server/lib/elastic-search/elastic-search-playlists.ts

@@ -1,10 +1,11 @@
+import { MappingProperty, PropertyName } from '@elastic/elasticsearch/lib/api/types'
 import { elasticSearch } from '../../helpers/elastic-search'
 import { logger } from '../../helpers/logger'
 import { buildUrl } from '../../helpers/utils'
 import { CONFIG, ELASTIC_SEARCH_QUERY } from '../../initializers/constants'
 import { DBPlaylist, EnhancedPlaylist, IndexablePlaylist } from '../../types/playlist.model'
 import { PlaylistsSearchQuery } from '../../types/search-query/playlist-search.model'
-import { buildSort, extractQueryResult } from './elastic-search-queries'
+import { buildSort, extractSearchQueryResult } from './elastic-search-queries'
 import { addUUIDFilters, buildMultiMatchBool } from './shared'
 import { buildChannelOrAccountSummaryMapping, formatActorForDB, formatActorSummaryForAPI } from './shared/elastic-search-actor'
 
@@ -67,7 +68,7 @@ async function queryPlaylists (search: PlaylistsSearchQuery) {
     body
   })
 
-  return extractQueryResult(res)
+  return extractSearchQueryResult(res)
 }
 
 function formatPlaylistForAPI (p: DBPlaylist, fromHost?: string): EnhancedPlaylist {
@@ -216,7 +217,7 @@ function buildPlaylistsMapping () {
     videoChannel: {
       properties: buildChannelOrAccountSummaryMapping()
     }
-  }
+  } as Record<PropertyName, MappingProperty>
 }
 
 export {

server/lib/elastic-search/elastic-search-queries.ts

@@ -1,5 +1,6 @@
 import { difference } from 'lodash'
-import { ApiResponse } from '@elastic/elasticsearch'
+import { estypes } from '@elastic/elasticsearch'
+import { AggregationsBuckets, AggregationsStringTermsAggregate, AggregationsStringTermsBucket } from '@elastic/elasticsearch/lib/api/types'
 import { elasticSearch } from '../../helpers/elastic-search'
 import { logger } from '../../helpers/logger'
 
@@ -10,7 +11,7 @@ async function removeNotExistingIdsFromHost (indexName: string, host: string, ex
 
   logger.info({ idsToRemove }, 'Will remove %d entries from %s of host %s.', idsToRemove.length, indexName, host)
 
-  return elasticSearch.delete_by_query({
+  return elasticSearch.deleteByQuery({
     index: indexName,
     body: {
       query: {
@@ -38,7 +39,7 @@ function removeFromHosts (indexName: string, hosts: string[]) {
 
   logger.info({ hosts }, 'Will remove entries of index %s from hosts.', indexName)
 
-  return elasticSearch.delete_by_query({
+  return elasticSearch.deleteByQuery({
     index: indexName,
     body: {
       query: {
@@ -55,7 +56,7 @@ function removeFromHosts (indexName: string, hosts: string[]) {
 }
 
 async function getIdsOf (indexName: string, host: string) {
-  const res = await elasticSearch.search({
+  const res = await elasticSearch.search<unknown, Record<'ids', AggregationsStringTermsAggregate>>({
     index: indexName,
     body: {
       size: 0,
@@ -81,13 +82,21 @@ async function getIdsOf (indexName: string, host: string) {
     }
   })
 
-  return res.body.aggregations.ids.buckets.map(b => b.key)
+  return extractBucketsFromAggregation<number>(res.aggregations.ids.buckets).map(b => b.key)
 }
 
-function extractQueryResult (result: ApiResponse<any, any>) {
-  const hits = result.body.hits
+function extractSearchQueryResult (result: estypes.SearchResponse<any, any>) {
+  const hits = result.hits
 
-  return { total: hits.total.value, data: hits.hits.map(h => Object.assign(h._source, { score: h._score })) }
+  return {
+    total: (hits.total as estypes.SearchTotalHits).value,
+    data: hits.hits.map(h => Object.assign(h._source, { score: h._score }))
+  }
+}
+
+function extractBucketsFromAggregation <T extends string | number> (buckets: AggregationsBuckets<AggregationsStringTermsBucket>) {
+  // FIXME: key returned by elastic search can also be a number
+  return buckets as unknown as { key: T }[]
 }
 
 function buildSort (value: string) {
@@ -117,7 +126,8 @@ export {
   elasticSearch,
   removeNotExistingIdsFromHost,
   getIdsOf,
-  extractQueryResult,
+  extractSearchQueryResult,
   removeFromHosts,
-  buildSort
+  buildSort,
+  extractBucketsFromAggregation
 }

server/lib/elastic-search/elastic-search-videos.ts

@@ -1,3 +1,4 @@
+import { MappingProperty, PropertyName } from '@elastic/elasticsearch/lib/api/types'
 import { exists } from '../../helpers/custom-validators/misc'
 import { elasticSearch } from '../../helpers/elastic-search'
 import { logger } from '../../helpers/logger'
@@ -5,7 +6,7 @@ import { buildUrl } from '../../helpers/utils'
 import { CONFIG, ELASTIC_SEARCH_QUERY } from '../../initializers/constants'
 import { VideosSearchQuery } from '../../types/search-query/video-search.model'
 import { DBVideo, DBVideoDetails, EnhancedVideo, IndexableVideo, IndexableVideoDetails } from '../../types/video.model'
-import { buildSort, extractQueryResult } from './elastic-search-queries'
+import { buildSort, extractSearchQueryResult } from './elastic-search-queries'
 import { addUUIDFilters, buildMultiMatchBool } from './shared'
 import { buildChannelOrAccountSummaryMapping, formatActorForDB, formatActorSummaryForAPI } from './shared/elastic-search-actor'
 
@@ -225,7 +226,7 @@ async function queryVideos (search: VideosSearchQuery) {
     body
   })
 
-  return extractQueryResult(res)
+  return extractSearchQueryResult(res)
 }
 
 function buildVideosMapping () {
@@ -368,7 +369,7 @@ function buildVideosMapping () {
     channel: {
       properties: buildChannelOrAccountSummaryMapping()
     }
-  }
+  } as Record<PropertyName, MappingProperty>
 }
 
 function formatVideoForDB (v: IndexableVideo | IndexableVideoDetails): DBVideo | DBVideoDetails {

server/lib/elastic-search/shared/elastic-search-actor.ts

@@ -1,3 +1,4 @@
+import { MappingProperty, PropertyName } from '@elastic/elasticsearch/lib/api/types'
 import { AccountSummary, VideoChannelSummary } from '../../../../PeerTube/shared/models'
 import { AdditionalActorAttributes } from '../../../types/actor.model'
 import { formatActorImageForDB } from './'
@@ -38,7 +39,7 @@ function buildChannelOrAccountSummaryMapping () {
     avatars: {
       properties: buildActorImageMapping()
     }
-  }
+  } as Record<PropertyName, MappingProperty>
 }
 
 function buildChannelOrAccountCommonMapping () {
@@ -64,7 +65,7 @@ function buildChannelOrAccountCommonMapping () {
     description: {
       type: 'text'
     }
-  }
+  } as Record<PropertyName, MappingProperty>
 }
 
 function formatActorSummaryForAPI (actor: (AccountSummary | VideoChannelSummary) & AdditionalActorAttributes) {

server/lib/elastic-search/shared/elastic-search-avatar.ts

@@ -1,3 +1,4 @@
+import { MappingProperty, PropertyName } from '@elastic/elasticsearch/lib/api/types'
 import { ActorImage } from '../../../../PeerTube/shared/models'
 import { buildUrl } from '../../../helpers/utils'
 
@@ -62,7 +63,7 @@ function buildActorImageMapping () {
       type: 'date',
       format: 'date_optional_time'
     }
-  }
+  } as Record<PropertyName, MappingProperty>
 }
 
 export {

server/lib/elastic-search/shared/query-helpers.ts

@@ -1,6 +1,7 @@
-import { ELASTIC_SEARCH_QUERY } from '../../../initializers/constants'
 import validator from 'validator'
 
+import { ELASTIC_SEARCH_QUERY } from '../../../initializers/constants'
+
 function addUUIDFilters (filters: any[], uuids: string[]) {
   if (!filters) return
 

server/lib/indexers/shared/abstract-indexer.ts

@@ -1,5 +1,6 @@
 import { AsyncQueue, queue } from 'async'
 import { inspect } from 'util'
+import { MappingProperty, PropertyName } from '@elastic/elasticsearch/lib/api/types'
 import { logger } from '../../../helpers/logger'
 import { INDEXER_QUEUE_CONCURRENCY } from '../../../initializers/constants'
 import { buildIndex, indexDocuments, refreshIndex } from '../../../lib/elastic-search/elastic-search-index'
@@ -13,7 +14,7 @@ export abstract class AbstractIndexer <T extends IndexableDoc, DB> {
   protected readonly indexQueue: AsyncQueue<QueueParam>
 
   abstract indexSpecificElement (host: string, uuid: string): Promise<any>
-  abstract buildMapping (): object
+  abstract buildMapping (): Record<PropertyName, MappingProperty>
 
   constructor (
     protected readonly indexName: string,

server/lib/schedulers/abstract-scheduler.ts

@@ -1,5 +1,5 @@
 import { logger } from '../../helpers/logger'
-import * as Bluebird from 'bluebird'
+import Bluebird from 'bluebird'
 import { inspect } from 'util'
 
 export abstract class AbstractScheduler {

server/lib/schedulers/indexation-scheduler.ts

@@ -1,4 +1,4 @@
-import * as Bluebird from 'bluebird'
+import Bluebird from 'bluebird'
 import { IndexablePlaylist } from 'server/types/playlist.model'
 import { inspect } from 'util'
 import { logger } from '../../helpers/logger'

server/middlewares/pagination.ts

@@ -1,5 +1,5 @@
 import 'express-validator'
-import * as express from 'express'
+import express from 'express'
 import { PAGINATION_COUNT } from '../initializers/constants'
 
 function setDefaultPagination (req: express.Request, res: express.Response, next: express.NextFunction) {

server/middlewares/sort.ts

@@ -1,4 +1,4 @@
-import * as express from 'express'
+import express from 'express'
 
 const setDefaultSort = setDefaultSortFactory('-createdAt')
 

server/middlewares/validators/method.ts

@@ -1,4 +1,4 @@
-import * as express from 'express'
+import express from 'express'
 
 const methodsValidator = (methods: string[]) => {
   return (req: express.Request, res: express.Response, next: express.NextFunction) => {

server/middlewares/validators/pagination.ts

@@ -1,4 +1,4 @@
-import * as express from 'express'
+import express from 'express'
 import { check } from 'express-validator'
 import { PAGINATION_COUNT, PAGINATION_START } from '../../initializers/constants'
 import { areValidationErrors } from './utils'

server/middlewares/validators/search.ts

@@ -1,4 +1,4 @@
-import * as express from 'express'
+import express from 'express'
 import { check } from 'express-validator'
 import { isDateValid, toArray } from '../../helpers/custom-validators/misc'
 import { isNSFWQueryValid, isNumberArray, isStringArray } from '../../helpers/custom-validators/search-videos'

server/middlewares/validators/utils.ts

@@ -1,4 +1,4 @@
-import * as express from 'express'
+import express from 'express'
 import { check, validationResult } from 'express-validator'
 import { logger } from '../../helpers/logger'
 

tsconfig.json

@@ -6,6 +6,7 @@
     "sourceMap": false,
     "experimentalDecorators": true,
     "emitDecoratorMetadata": true,
+    "esModuleInterop": true,
     "outDir": "./dist",
     "lib": [
       "dom",