Peertube/sepia-search-motore-di-ricerca-video
1
0
Fork 0
mirror of https://framagit.org/framasoft/peertube/search-index/ synced 2024-12-17 19:58:38 +01:00
Code Issues Releases Wiki Activity

Fix removing non existing hosts

Browse Source
This commit is contained in:
Chocobozzz 2024-01-09 11:06:15 +01:00
parent bf63503e23
commit aaad2ea75d
No known key found for this signature in database
GPG Key ID: 583A612D890159BE
3 changed files with 8 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
server/lib/indexers/shared/abstract-indexer.ts
View File

@ -74,9 +74,9 @@ export abstract class AbstractIndexer <T extends IndexableDoc, DB> {
}) })
} }
removeFromHosts (hosts: string[]) { removeFromHosts (existingHosts: string[]) {
return client.index(this.indexName).deleteDocuments({ return client.index(this.indexName).deleteDocuments({
filter: 'host IN ' + buildInValuesArray(Array.from(hosts)) filter: 'host NOT IN ' + buildInValuesArray(Array.from(existingHosts))
}) })
} }

36
server/lib/meilisearch/meilisearch-instances.ts
View File

@ -1,10 +1,9 @@
import { CONFIG } from '../../initializers/constants' import { CONFIG } from '../../initializers/constants'
import { listIndexInstancesHost, getMajorInstanceVersion } from '../requests/instances-index' import { listIndexInstancesHost, getMajorInstanceVersion } from '../requests/instances-index'
import { client } from '../../helpers/meilisearch'
import { logger } from '../../helpers/logger' import { logger } from '../../helpers/logger'
import Bluebird from 'bluebird' import Bluebird from 'bluebird'
async function buildInstanceHosts () { export async function buildInstanceHosts () {
let indexHosts = await listIndexInstancesHost() let indexHosts = await listIndexInstancesHost()
if (CONFIG.INSTANCES_INDEX.WHITELIST.ENABLED) { if (CONFIG.INSTANCES_INDEX.WHITELIST.ENABLED) {
@ -27,36 +26,5 @@ async function buildInstanceHosts () {
return true return true
}, { concurrency: 10 }) }, { concurrency: 10 })
const dbHosts = await listDBInstances() return indexHosts
const removedHosts = getRemovedHosts(dbHosts, indexHosts)
return { indexHosts, removedHosts }
}
export {
buildInstanceHosts
}
// ##################################################
async function listDBInstances () {
const setResult = new Set<string>()
const indexes = [
CONFIG.MEILISEARCH.INDEXES.VIDEOS,
CONFIG.MEILISEARCH.INDEXES.CHANNELS
]
for (const index of indexes) {
const result = await client.index(index).searchForFacetValues({ facetName: 'host' })
for (const b of result.facetHits) {
setResult.add(b.value)
}
}
return Array.from(setResult)
}
function getRemovedHosts (dbHosts: string[], indexHosts: string[]) {
return dbHosts.filter(dbHost => indexHosts.includes(dbHost) === false)
} }

11
server/lib/schedulers/indexation-scheduler.ts
View File

@ -56,18 +56,15 @@ export class IndexationScheduler extends AbstractScheduler {
private async runIndexer () { private async runIndexer () {
logger.info('Running indexer.') logger.info('Running indexer.')
const { indexHosts, removedHosts } = await buildInstanceHosts() this.indexedHosts = await buildInstanceHosts()
this.indexedHosts = indexHosts
logger.info({ removedHosts }, `Will remove ${removedHosts.length} hosts that do not exist anymore`) logger.info({ indexHosts: this.indexedHosts }, `Will index ${this.indexedHosts.length} hosts and remove non existing hosts`)
for (const o of this.indexers) { for (const o of this.indexers) {
await o.removeFromHosts(removedHosts) await o.removeFromHosts(this.indexedHosts)
} }
logger.info({ indexHosts }, `Will index ${indexHosts.length} hosts`) await Bluebird.map(this.indexedHosts, async host => {
await Bluebird.map(indexHosts, async host => {
try { try {
await this.indexHost(host) await this.indexHost(host)
} catch (err) { } catch (err) {