Peertube
/
sepia-search-motore-di-ricerca-video
mirror of https://framagit.org/framasoft/peertube/search-index/
1
0
Fork 0
Code Issues Releases Wiki Activity

Put some constants in config

This commit is contained in:
Chocobozzz 2024-01-04 16:50:17 +01:00
parent 491f936906
commit 65295a804a
No known key found for this signature in database
GPG Key ID: 583A612D890159BE
4 changed files with 15 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
config/default.yaml
View File

@ -54,3 +54,10 @@ api:
enabled: false enabled: false
# Array of hosts # Array of hosts
hosts: null hosts: null
indexer:
# How many hosts in parallel to index
host_concurrency: 10
# How much time to wait before bulk indexing in Meilisearch data
bulk_indexation_interval_ms: 10000

9
server/initializers/constants.ts
View File

@ -45,6 +45,10 @@ const CONFIG = {
ENABLED: config.get<boolean>('api.blacklist.enabled'), ENABLED: config.get<boolean>('api.blacklist.enabled'),
HOSTS: config.get<string[]>('api.blacklist.hosts') HOSTS: config.get<string[]>('api.blacklist.hosts')
} }
},
INDEXER: {
HOST_CONCURRENCY: config.get<number>('indexer.host_concurrency'),
BULK_INDEXATION_INTERVAL_MS: config.get<number>('indexer.bulk_indexation_interval_ms')
} }
} }
@ -70,11 +74,8 @@ const SCHEDULER_INTERVALS_MS = {
const INDEXER_COUNT = 20 const INDEXER_COUNT = 20
const INDEXER_LIMIT = 500000 const INDEXER_LIMIT = 500000
const INDEXER_HOST_CONCURRENCY = 3
const INDEXER_QUEUE_CONCURRENCY = 3 const INDEXER_QUEUE_CONCURRENCY = 3
const INDEXER_BULK_INDEXATION_MS = 10000
const REQUESTS = { const REQUESTS = {
MAX_RETRIES: 10, MAX_RETRIES: 10,
WAIT: 10000 // 10 seconds WAIT: 10000 // 10 seconds
@ -102,9 +103,7 @@ export {
SORTABLE_COLUMNS, SORTABLE_COLUMNS,
INDEXER_QUEUE_CONCURRENCY, INDEXER_QUEUE_CONCURRENCY,
SCHEDULER_INTERVALS_MS, SCHEDULER_INTERVALS_MS,
INDEXER_HOST_CONCURRENCY,
INDEXER_COUNT, INDEXER_COUNT,
INDEXER_LIMIT, INDEXER_LIMIT,
INDEXER_BULK_INDEXATION_MS,
REQUESTS REQUESTS
} }

4
server/lib/indexers/shared/abstract-indexer.ts
View File

@ -1,7 +1,7 @@
import { QueueObject, queue } from 'async' import { QueueObject, queue } from 'async'
import { inspect } from 'util' import { inspect } from 'util'
import { logger } from '../../../helpers/logger' import { logger } from '../../../helpers/logger'
import { INDEXER_BULK_INDEXATION_MS, INDEXER_QUEUE_CONCURRENCY } from '../../../initializers/constants' import { CONFIG, INDEXER_QUEUE_CONCURRENCY } from '../../../initializers/constants'
import { IndexableDoc } from '../../../types/indexable-doc.model' import { IndexableDoc } from '../../../types/indexable-doc.model'
import { client } from '../../../helpers/meilisearch' import { client } from '../../../helpers/meilisearch'
import { buildInValuesArray } from '../../meilisearch/meilisearch-queries' import { buildInValuesArray } from '../../meilisearch/meilisearch-queries'
@ -117,7 +117,7 @@ export abstract class AbstractIndexer <T extends IndexableDoc, DB> {
} catch (err) { } catch (err) {
logger.error({ err }, 'Cannot schedule bulk indexation') logger.error({ err }, 'Cannot schedule bulk indexation')
} }
}, INDEXER_BULK_INDEXATION_MS) }, CONFIG.INDEXER.BULK_INDEXATION_INTERVAL_MS)
} }
private async indexElements (elements: T[]) { private async indexElements (elements: T[]) {

4
server/lib/schedulers/indexation-scheduler.ts
View File

@ -1,7 +1,7 @@
import Bluebird from 'bluebird' import Bluebird from 'bluebird'
import { inspect } from 'util' import { inspect } from 'util'
import { logger } from '../../helpers/logger' import { logger } from '../../helpers/logger'
import { INDEXER_HOST_CONCURRENCY, INDEXER_COUNT, INDEXER_LIMIT, SCHEDULER_INTERVALS_MS } from '../../initializers/constants' import { INDEXER_COUNT, INDEXER_LIMIT, SCHEDULER_INTERVALS_MS, CONFIG } from '../../initializers/constants'
import { IndexableVideo } from '../../types/video.model' import { IndexableVideo } from '../../types/video.model'
import { buildInstanceHosts } from '../meilisearch/meilisearch-instances' import { buildInstanceHosts } from '../meilisearch/meilisearch-instances'
import { ChannelIndexer } from '../indexers/channel-indexer' import { ChannelIndexer } from '../indexers/channel-indexer'
@ -72,7 +72,7 @@ export class IndexationScheduler extends AbstractScheduler {
console.error(inspect(err, { depth: 10 })) console.error(inspect(err, { depth: 10 }))
logger.warn({ err: inspect(err) }, 'Cannot index videos from %s.', host) logger.warn({ err: inspect(err) }, 'Cannot index videos from %s.', host)
} }
}, { concurrency: INDEXER_HOST_CONCURRENCY }) }, { concurrency: CONFIG.INDEXER.HOST_CONCURRENCY })
logger.info('Indexer ended.') logger.info('Indexer ended.')
} }