Put some constants in config

config/default.yaml

@@ -54,3 +54,10 @@ api:
     enabled: false
     # Array of hosts
     hosts: null
+
+indexer:
+  # How many hosts in parallel to index
+  host_concurrency: 10
+
+  # How much time to wait before bulk indexing in Meilisearch data
+  bulk_indexation_interval_ms: 10000

server/initializers/constants.ts

@@ -45,6 +45,10 @@ const CONFIG = {
       ENABLED: config.get<boolean>('api.blacklist.enabled'),
       HOSTS: config.get<string[]>('api.blacklist.hosts')
     }
+  },
+  INDEXER: {
+    HOST_CONCURRENCY: config.get<number>('indexer.host_concurrency'),
+    BULK_INDEXATION_INTERVAL_MS: config.get<number>('indexer.bulk_indexation_interval_ms')
   }
 }
 
@@ -70,11 +74,8 @@ const SCHEDULER_INTERVALS_MS = {
 const INDEXER_COUNT = 20
 const INDEXER_LIMIT = 500000
 
-const INDEXER_HOST_CONCURRENCY = 3
 const INDEXER_QUEUE_CONCURRENCY = 3
 
-const INDEXER_BULK_INDEXATION_MS = 10000
-
 const REQUESTS = {
   MAX_RETRIES: 10,
   WAIT: 10000 // 10 seconds
@@ -102,9 +103,7 @@ export {
   SORTABLE_COLUMNS,
   INDEXER_QUEUE_CONCURRENCY,
   SCHEDULER_INTERVALS_MS,
-  INDEXER_HOST_CONCURRENCY,
   INDEXER_COUNT,
   INDEXER_LIMIT,
-  INDEXER_BULK_INDEXATION_MS,
   REQUESTS
 }

server/lib/indexers/shared/abstract-indexer.ts

@@ -1,7 +1,7 @@
 import { QueueObject, queue } from 'async'
 import { inspect } from 'util'
 import { logger } from '../../../helpers/logger'
-import { INDEXER_BULK_INDEXATION_MS, INDEXER_QUEUE_CONCURRENCY } from '../../../initializers/constants'
+import { CONFIG, INDEXER_QUEUE_CONCURRENCY } from '../../../initializers/constants'
 import { IndexableDoc } from '../../../types/indexable-doc.model'
 import { client } from '../../../helpers/meilisearch'
 import { buildInValuesArray } from '../../meilisearch/meilisearch-queries'
@@ -117,7 +117,7 @@ export abstract class AbstractIndexer <T extends IndexableDoc, DB> {
       } catch (err) {
         logger.error({ err }, 'Cannot schedule bulk indexation')
       }
-    }, INDEXER_BULK_INDEXATION_MS)
+    }, CONFIG.INDEXER.BULK_INDEXATION_INTERVAL_MS)
   }
 
   private async indexElements (elements: T[]) {

server/lib/schedulers/indexation-scheduler.ts

@@ -1,7 +1,7 @@
 import Bluebird from 'bluebird'
 import { inspect } from 'util'
 import { logger } from '../../helpers/logger'
-import { INDEXER_HOST_CONCURRENCY, INDEXER_COUNT, INDEXER_LIMIT, SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
+import { INDEXER_COUNT, INDEXER_LIMIT, SCHEDULER_INTERVALS_MS, CONFIG } from '../../initializers/constants'
 import { IndexableVideo } from '../../types/video.model'
 import { buildInstanceHosts } from '../meilisearch/meilisearch-instances'
 import { ChannelIndexer } from '../indexers/channel-indexer'
@@ -72,7 +72,7 @@ export class IndexationScheduler extends AbstractScheduler {
         console.error(inspect(err, { depth: 10 }))
         logger.warn({ err: inspect(err) }, 'Cannot index videos from %s.', host)
       }
-    }, { concurrency: INDEXER_HOST_CONCURRENCY })
+    }, { concurrency: CONFIG.INDEXER.HOST_CONCURRENCY })
 
     logger.info('Indexer ended.')
   }