Peertube
/
sepia-search-motore-di-ricerca-video
mirror of https://framagit.org/framasoft/peertube/search-index/
1
0
Fork 0
Code Issues Releases Wiki Activity

Add blockedAccounts/blockedHosts support

This commit is contained in:
Chocobozzz 2020-05-28 16:32:49 +02:00
parent 3c603d2324
commit 0feb8973a8
No known key found for this signature in database
GPG Key ID: 583A612D890159BE
20 changed files with 134 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
PeerTube

@ -1 +1 @@
Subproject commit 68ca61941e3a7ca4c018566d2c496afd27dbd76d
Subproject commit 99139e7753e20ab0fba8eae5638d3dd3e792fe43

1
package.json
View File

@ -48,6 +48,7 @@
"@types/body-parser": "^1.16.3",
"@types/config": "^0.0.36",
"@types/express": "^4.16.1",
"@types/fluent-ffmpeg": "^2.1.14",
"@types/lodash": "^4.14.149",
"@types/mkdirp": "^1.0.0",
"@types/morgan": "^1.7.32",

2
server.ts
View File

@ -49,7 +49,7 @@ app.use(function (err, req, res, next) {
error = err.stack || err.message || err
}
logger.error('Error in controller.', { error })
logger.error({ error }, 'Error in controller.')
return res.status(err.status || 500).end()
})

3
server/controllers/api/search-channels.ts
View File

@ -5,7 +5,7 @@ import { asyncMiddleware } from '../../middlewares/async'
import { setDefaultPagination } from '../../middlewares/pagination'
import { setDefaultSearchSort } from '../../middlewares/sort'
import { paginationValidator } from '../../middlewares/validators/pagination'
import { videoChannelsSearchValidator } from '../../middlewares/validators/search'
import { videoChannelsSearchValidator, commonFiltersValidators } from '../../middlewares/validators/search'
import { channelsSearchSortValidator } from '../../middlewares/validators/sort'
const searchChannelsRouter = express.Router()
@ -15,6 +15,7 @@ searchChannelsRouter.get('/search/video-channels',
setDefaultPagination,
channelsSearchSortValidator,
setDefaultSearchSort,
commonFiltersValidators,
videoChannelsSearchValidator,
asyncMiddleware(searchChannels)
)

10
server/controllers/api/search-videos.ts
View File

@ -4,8 +4,9 @@ import { setDefaultPagination } from '../../middlewares/pagination'
import { asyncMiddleware } from '../../middlewares/async'
import { formatVideoForAPI, queryVideos } from '../../lib/elastic-search-videos'
import { videosSearchSortValidator } from '../../middlewares/validators/sort'
import { commonVideosFiltersValidator, videosSearchValidator } from '../../middlewares/validators/search'
import { commonVideosFiltersValidator, videosSearchValidator, commonFiltersValidators } from '../../middlewares/validators/search'
import { setDefaultSearchSort } from '../../middlewares/sort'
import { VideosSearchQuery } from 'server/types/video-search.model'
const searchVideosRouter = express.Router()
@ -14,6 +15,7 @@ searchVideosRouter.get('/search/videos',
setDefaultPagination,
videosSearchSortValidator,
setDefaultSearchSort,
commonFiltersValidators,
commonVideosFiltersValidator,
videosSearchValidator,
asyncMiddleware(searchVideos)
@ -26,10 +28,12 @@ export { searchVideosRouter }
// ---------------------------------------------------------------------------
async function searchVideos (req: express.Request, res: express.Response) {
const resultList = await queryVideos(req.query)
const query = req.query as VideosSearchQuery
const resultList = await queryVideos(query)
return res.json({
total: resultList.total,
data: resultList.data.map(v => formatVideoForAPI(v, req.query.fromHost as string))
data: resultList.data.map(v => formatVideoForAPI(v, query.fromHost))
})
}

2
server/helpers/elastic-search.ts
View File

@ -93,7 +93,7 @@ async function indexDocuments <T extends IndexableDoc> (options: {
if (resultBody.errors === true) {
const msg = 'Cannot insert data in elastic search.'
logger.error(msg, { err: resultBody })
logger.error({ err: resultBody }, msg)
throw new Error(msg)
}

33
server/lib/elastic-search-channels.ts
View File

@ -29,7 +29,7 @@ async function removeNotExistingChannels (host: string, existingChannels: Set<nu
const idsToRemove = difference(idsFromDB, Array.from(existingChannels))
logger.info('Will remove %d channels from %s.', idsToRemove.length, host, { idsToRemove })
logger.info({ idsToRemove }, 'Will remove %d channels from %s.', idsToRemove.length, host)
return elasticSearch.delete_by_query({
index: CONFIG.ELASTIC_SEARCH.INDEXES.CHANNELS,
@ -56,6 +56,7 @@ async function removeNotExistingChannels (host: string, existingChannels: Set<nu
async function queryChannels (search: ChannelsSearchQuery) {
const bool: any = {}
const mustNot: any[] = []
if (search.search) {
Object.assign(bool, {
@ -71,6 +72,26 @@ async function queryChannels (search: ChannelsSearchQuery) {
})
}
if (search.blockedAccounts) {
mustNot.push({
terms: {
'ownerAccount.handle': search.blockedAccounts
}
})
}
if (search.blockedHosts) {
mustNot.push({
terms: {
host: search.blockedHosts
}
})
}
if (mustNot.length !== 0) {
Object.assign(bool, { must_not: mustNot })
}
const body = {
from: search.start,
size: search.count,
@ -78,7 +99,7 @@ async function queryChannels (search: ChannelsSearchQuery) {
query: { bool }
}
logger.debug('Will query Elastic Search for channels.', { body })
logger.debug({ body }, 'Will query Elastic Search for channels.')
const res = await elasticSearch.search({
index: CONFIG.ELASTIC_SEARCH.INDEXES.CHANNELS,
@ -151,6 +172,8 @@ function formatChannelForDB (c: IndexableChannel): DBChannel {
description: c.description,
support: c.support,
handle: `${c.name}@${c.host}`,
ownerAccount: {
id: c.ownerAccount.id,
url: c.ownerAccount.url,
@ -164,6 +187,8 @@ function formatChannelForDB (c: IndexableChannel): DBChannel {
createdAt: c.ownerAccount.createdAt,
updatedAt: c.ownerAccount.updatedAt,
handle: `${c.ownerAccount.name}@${c.ownerAccount.host}`,
avatar: formatAvatarForDB(c.ownerAccount)
}
}
@ -228,6 +253,10 @@ function buildChannelOrAccountCommonMapping () {
type: 'keyword'
},
handle: {
type: 'keyword'
},
displayName: {
type: 'text'
},

34
server/lib/elastic-search-videos.ts
View File

@ -27,7 +27,7 @@ function refreshVideosIndex () {
function removeVideosFromHosts (hosts: string[]) {
if (hosts.length === 0) return
logger.info('Will remove videos from hosts.', { hosts })
logger.info({ hosts }, 'Will remove videos from hosts.')
return elasticSearch.delete_by_query({
index: CONFIG.ELASTIC_SEARCH.INDEXES.VIDEOS,
@ -50,7 +50,7 @@ async function removeNotExistingVideos (host: string, existingVideos: Set<number
const idsToRemove = difference(idsFromDB, Array.from(existingVideos))
logger.info('Will remove %d videos from %s.', idsToRemove.length, host, { idsToRemove })
logger.info({ idsToRemove }, 'Will remove %d videos from %s.', idsToRemove.length, host)
return elasticSearch.delete_by_query({
index: CONFIG.ELASTIC_SEARCH.INDEXES.VIDEOS,
@ -107,6 +107,7 @@ async function getVideoIdsOf (host: string) {
async function queryVideos (search: VideosSearchQuery) {
const bool: any = {}
const filter: any[] = []
const mustNot: any[] = []
if (search.search) {
Object.assign(bool, {
@ -122,6 +123,22 @@ async function queryVideos (search: VideosSearchQuery) {
})
}
if (search.blockedAccounts) {
mustNot.push({
terms: {
'account.handle': search.blockedAccounts
}
})
}
if (search.blockedHosts) {
mustNot.push({
terms: {
host: search.blockedHosts
}
})
}
if (search.startDate) {
filter.push({
range: {
@ -234,6 +251,10 @@ async function queryVideos (search: VideosSearchQuery) {
Object.assign(bool, { filter })
if (mustNot.length !== 0) {
Object.assign(bool, { must_not: mustNot })
}
const body = {
from: search.start,
size: search.count,
@ -241,7 +262,7 @@ async function queryVideos (search: VideosSearchQuery) {
query: { bool }
}
logger.debug('Will query Elastic Search for videos.', { body })
logger.debug({ body }, 'Will query Elastic Search for videos.')
const res = await elasticSearch.search({
index: CONFIG.ELASTIC_SEARCH.INDEXES.VIDEOS,
@ -314,6 +335,8 @@ function formatVideoForDB (v: IndexableVideo | IndexableVideoDetails): DBVideo |
url: v.account.url,
host: v.account.host,
handle: `${v.account.name}@${v.account.host}`,
avatar: formatAvatarForDB(v.account)
},
@ -324,6 +347,8 @@ function formatVideoForDB (v: IndexableVideo | IndexableVideoDetails): DBVideo |
url: v.channel.url,
host: v.channel.host,
handle: `${v.channel.name}@${v.channel.host}`,
avatar: formatAvatarForDB(v.channel)
}
}
@ -415,6 +440,9 @@ function buildChannelOrAccountMapping () {
host: {
type: 'keyword'
},
handle: {
type: 'keyword'
},
avatar: {
properties: buildAvatarMapping()

2
server/lib/schedulers/abstract-scheduler.ts
View File

@ -26,7 +26,7 @@ export abstract class AbstractScheduler {
try {
await this.internalExecute()
} catch (err) {
logger.error('Cannot execute %s scheduler.', this.constructor.name, { err: inspect(err) })
logger.error({ err: inspect(err) }, 'Cannot execute %s scheduler.', this.constructor.name)
} finally {
this.isRunning = false
}

6
server/lib/schedulers/videos-indexer.ts
View File

@ -29,7 +29,7 @@ export class VideosIndexer extends AbstractScheduler {
this.indexSpecificVideo(task.host, task.uuid)
.then(() => cb())
.catch(err => {
logger.error('Error in index specific video %s of %s.', task.uuid, task.host, { err: inspect(err) })
logger.error({ err: inspect(err) }, 'Error in index specific video %s of %s.', task.uuid, task.host)
cb()
})
}, INDEXER_QUEUE_CONCURRENCY)
@ -38,7 +38,7 @@ export class VideosIndexer extends AbstractScheduler {
this.indexSpecificChannel(task.host, task.name)
.then(() => cb())
.catch(err => {
logger.error('Error in index specific channel %s@%s.', task.name, task.host, { err: inspect(err) })
logger.error({ err: inspect(err) }, 'Error in index specific channel %s@%s.', task.name, task.host)
cb()
})
}, INDEXER_QUEUE_CONCURRENCY)
@ -72,7 +72,7 @@ export class VideosIndexer extends AbstractScheduler {
await this.indexHost(host)
} catch (err) {
console.error(inspect(err, { depth: 10 }))
logger.warn('Cannot index videos from %s.', host, { err })
logger.warn({ err: inspect(err) }, 'Cannot index videos from %s.', host)
}
}, { concurrency: INDEXER_CONCURRENCY })

2
server/middlewares/validators/pagination.ts
View File

@ -8,7 +8,7 @@ const paginationValidator = [
query('count').optional().isInt({ min: 0 }).withMessage('Should have a number count'),
(req: express.Request, res: express.Response, next: express.NextFunction) => {
logger.debug('Checking pagination parameters', { parameters: req.query })
logger.debug({ parameters: req.query }, 'Checking pagination parameters')
if (areValidationErrors(req, res)) return

26
server/middlewares/validators/search.ts
View File

@ -5,6 +5,25 @@ import { isNSFWQueryValid, isNumberArray, isStringArray } from '../../helpers/cu
import { logger } from '../../helpers/logger'
import { areValidationErrors } from './utils'
const commonFiltersValidators = [
query('blockedAccounts')
.optional()
.customSanitizer(toArray)
.custom(isStringArray).withMessage('Should have a valid blockedAccounts array'),
query('blockedHosts')
.optional()
.customSanitizer(toArray)
.custom(isStringArray).withMessage('Should have a valid hosts array'),
(req: express.Request, res: express.Response, next: express.NextFunction) => {
logger.debug({ parameters: req.query }, 'Checking commons filters query')
if (areValidationErrors(req, res)) return
return next()
}
]
const commonVideosFiltersValidator = [
query('categoryOneOf')
.optional()
@ -31,7 +50,7 @@ const commonVideosFiltersValidator = [
.custom(isNSFWQueryValid).withMessage('Should have a valid NSFW attribute'),
(req: express.Request, res: express.Response, next: express.NextFunction) => {
logger.debug('Checking commons video filters query', { parameters: req.query })
logger.debug({ parameters: req.query }, 'Checking commons video filters query')
if (areValidationErrors(req, res)) return
@ -52,7 +71,7 @@ const videosSearchValidator = [
query('durationMax').optional().isInt().withMessage('Should have a valid max duration'),
(req: express.Request, res: express.Response, next: express.NextFunction) => {
logger.debug('Checking videos search query', { parameters: req.query })
logger.debug({ parameters: req.query }, 'Checking videos search query')
if (areValidationErrors(req, res)) return
@ -64,7 +83,7 @@ const videoChannelsSearchValidator = [
query('search').not().isEmpty().withMessage('Should have a valid search'),
(req: express.Request, res: express.Response, next: express.NextFunction) => {
logger.debug('Checking video channels search query', { parameters: req.query })
logger.debug({ parameters: req.query }, 'Checking video channels search query')
if (areValidationErrors(req, res)) return
@ -76,6 +95,7 @@ const videoChannelsSearchValidator = [
export {
videoChannelsSearchValidator,
commonFiltersValidators,
commonVideosFiltersValidator,
videosSearchValidator
}

2
server/middlewares/validators/sort.ts
View File

@ -5,7 +5,7 @@ const SORTABLE_VIDEOS_SEARCH_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VI
const SORTABLE_CHANNELS_SEARCH_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.CHANNELS_SEARCH)
const videosSearchSortValidator = checkSort(SORTABLE_VIDEOS_SEARCH_COLUMNS)
const channelsSearchSortValidator = checkSort(SORTABLE_VIDEOS_SEARCH_COLUMNS)
const channelsSearchSortValidator = checkSort(SORTABLE_CHANNELS_SEARCH_COLUMNS)
// ---------------------------------------------------------------------------

4
server/middlewares/validators/utils.ts
View File

@ -6,7 +6,7 @@ function areValidationErrors (req: express.Request, res: express.Response) {
const errors = validationResult(req)
if (!errors.isEmpty()) {
logger.warn('Incorrect request parameters', { path: req.originalUrl, err: errors.mapped() })
logger.warn({ path: req.originalUrl, err: errors.mapped() }, 'Incorrect request parameters')
res.status(400).json({ errors: errors.mapped() })
return true
@ -20,7 +20,7 @@ function checkSort (sortableColumns: string[]) {
query('sort').optional().isIn(sortableColumns).withMessage('Should have correct sortable column'),
(req: express.Request, res: express.Response, next: express.NextFunction) => {
logger.debug('Checking sort parameters', { parameters: req.query })
logger.debug({ parameters: req.query }, 'Checking sort parameters')
if (areValidationErrors(req, res)) return

3
server/types/channel-search.model.ts
View File

@ -1,5 +1,6 @@
import {
VideoChannelsSearchQuery as PeerTubeChannelsSearchQuery
} from '../../PeerTube/shared/models/search/video-channels-search-query.model'
import { CommonSearch } from './common-search.model'
export type ChannelsSearchQuery = PeerTubeChannelsSearchQuery & { fromHost?: string }
export type ChannelsSearchQuery = PeerTubeChannelsSearchQuery & CommonSearch

4
server/types/channel.model.ts
View File

@ -1,5 +1,6 @@
import { IndexableDoc } from './elastic-search.model'
import { VideoChannel, VideoChannelSummary } from '@shared/models'
import { Account } from '@shared/models/actors/account.model'
export interface IndexableChannelSummary extends VideoChannelSummary, IndexableDoc {
}
@ -9,6 +10,9 @@ export interface IndexableChannel extends VideoChannel, IndexableDoc {
export interface DBChannel extends Omit<VideoChannel, 'isLocal'> {
indexedAt: Date
handle: string
ownerAccount?: Account & { handle: string }
}
export interface DBChannelSummary extends VideoChannelSummary {

5
server/types/common-search.model.ts Normal file
View File

@ -0,0 +1,5 @@
export type CommonSearch = {
blockedAccounts?: string[]
blockedHosts?: string[]
fromHost?: string
}

3
server/types/video-search.model.ts
View File

@ -1,3 +1,4 @@
import { VideosSearchQuery as PeerTubeVideosSearchQuery} from '../../PeerTube/shared/models/search/videos-search-query.model'
import { CommonSearch } from './common-search.model'
export type VideosSearchQuery = Omit<PeerTubeVideosSearchQuery, 'skipCount' | 'filter'>
export type VideosSearchQuery = Omit<PeerTubeVideosSearchQuery, 'skipCount' | 'filter'> & CommonSearch

8
server/types/video.model.ts
View File

@ -1,3 +1,5 @@
import { VideoChannel, VideoChannelSummary } from '@shared/models/videos/channel/video-channel.model'
import { Account, AccountSummary } from '@shared/models/actors/account.model'
import { Video, VideoDetails } from '@shared/models/videos/video.model'
import { IndexableDoc } from './elastic-search.model'
@ -10,9 +12,15 @@ export interface IndexableVideoDetails extends VideoDetails, IndexableDoc {
export interface DBVideoDetails extends Omit<VideoDetails, 'isLocal'> {
indexedAt: Date
host: string
account: Account & { handle: string }
channel: VideoChannel & { handle: string }
}
export interface DBVideo extends Omit<Video, 'isLocal'> {
indexedAt: Date
host: string
account: AccountSummary & { handle: string }
channel: VideoChannelSummary & { handle: string }
}

7
yarn.lock
View File

@ -110,6 +110,13 @@
"@types/qs" "*"
"@types/serve-static" "*"
"@types/fluent-ffmpeg@^2.1.14":
version "2.1.14"
resolved "https://registry.yarnpkg.com/@types/fluent-ffmpeg/-/fluent-ffmpeg-2.1.14.tgz#b21d60267fe269c2ea81fa3238a36a8349f8f2f3"
integrity sha512-nJrAX9ODNI7mUB0b7Y0Stx1a6dOpV3zXsOnWoBuEd9/woQhepBNCMeCyOL6SLJD3jn5sLw5ciDGH0RwJenCoag==
dependencies:
"@types/node" "*"
"@types/json-schema@^7.0.3":
version "7.0.4"
resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.4.tgz#38fd73ddfd9b55abb1e1b2ed578cb55bd7b7d339"