diff --git a/config/default.yaml b/config/default.yaml index 137a2f8..1170276 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -54,3 +54,10 @@ api: enabled: false # Array of hosts hosts: null + +indexer: + # How many hosts in parallel to index + host_concurrency: 10 + + # How much time to wait before bulk indexing in Meilisearch data + bulk_indexation_interval_ms: 10000 diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 4458bb1..3783659 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -45,6 +45,10 @@ const CONFIG = { ENABLED: config.get('api.blacklist.enabled'), HOSTS: config.get('api.blacklist.hosts') } + }, + INDEXER: { + HOST_CONCURRENCY: config.get('indexer.host_concurrency'), + BULK_INDEXATION_INTERVAL_MS: config.get('indexer.bulk_indexation_interval_ms') } } @@ -70,11 +74,8 @@ const SCHEDULER_INTERVALS_MS = { const INDEXER_COUNT = 20 const INDEXER_LIMIT = 500000 -const INDEXER_HOST_CONCURRENCY = 3 const INDEXER_QUEUE_CONCURRENCY = 3 -const INDEXER_BULK_INDEXATION_MS = 10000 - const REQUESTS = { MAX_RETRIES: 10, WAIT: 10000 // 10 seconds @@ -102,9 +103,7 @@ export { SORTABLE_COLUMNS, INDEXER_QUEUE_CONCURRENCY, SCHEDULER_INTERVALS_MS, - INDEXER_HOST_CONCURRENCY, INDEXER_COUNT, INDEXER_LIMIT, - INDEXER_BULK_INDEXATION_MS, REQUESTS } diff --git a/server/lib/indexers/shared/abstract-indexer.ts b/server/lib/indexers/shared/abstract-indexer.ts index 8940733..9bdf41b 100644 --- a/server/lib/indexers/shared/abstract-indexer.ts +++ b/server/lib/indexers/shared/abstract-indexer.ts @@ -1,7 +1,7 @@ import { QueueObject, queue } from 'async' import { inspect } from 'util' import { logger } from '../../../helpers/logger' -import { INDEXER_BULK_INDEXATION_MS, INDEXER_QUEUE_CONCURRENCY } from '../../../initializers/constants' +import { CONFIG, INDEXER_QUEUE_CONCURRENCY } from '../../../initializers/constants' import { IndexableDoc } from '../../../types/indexable-doc.model' import { client } from '../../../helpers/meilisearch' import { buildInValuesArray } from '../../meilisearch/meilisearch-queries' @@ -117,7 +117,7 @@ export abstract class AbstractIndexer { } catch (err) { logger.error({ err }, 'Cannot schedule bulk indexation') } - }, INDEXER_BULK_INDEXATION_MS) + }, CONFIG.INDEXER.BULK_INDEXATION_INTERVAL_MS) } private async indexElements (elements: T[]) { diff --git a/server/lib/schedulers/indexation-scheduler.ts b/server/lib/schedulers/indexation-scheduler.ts index eb34e0c..a7238e5 100644 --- a/server/lib/schedulers/indexation-scheduler.ts +++ b/server/lib/schedulers/indexation-scheduler.ts @@ -1,7 +1,7 @@ import Bluebird from 'bluebird' import { inspect } from 'util' import { logger } from '../../helpers/logger' -import { INDEXER_HOST_CONCURRENCY, INDEXER_COUNT, INDEXER_LIMIT, SCHEDULER_INTERVALS_MS } from '../../initializers/constants' +import { INDEXER_COUNT, INDEXER_LIMIT, SCHEDULER_INTERVALS_MS, CONFIG } from '../../initializers/constants' import { IndexableVideo } from '../../types/video.model' import { buildInstanceHosts } from '../meilisearch/meilisearch-instances' import { ChannelIndexer } from '../indexers/channel-indexer' @@ -72,7 +72,7 @@ export class IndexationScheduler extends AbstractScheduler { console.error(inspect(err, { depth: 10 })) logger.warn({ err: inspect(err) }, 'Cannot index videos from %s.', host) } - }, { concurrency: INDEXER_HOST_CONCURRENCY }) + }, { concurrency: CONFIG.INDEXER.HOST_CONCURRENCY }) logger.info('Indexer ended.') }