Peertube
/
sepia-search-motore-di-ricerca-video
mirror of https://framagit.org/framasoft/peertube/search-index/
1
0
Fork 0
Code Issues Releases Wiki Activity

Videos search works

This commit is contained in:
Chocobozzz 2020-02-18 15:33:21 +01:00
parent 8f26b09f11
commit e886f1c937
No known key found for this signature in database
GPG Key ID: 583A612D890159BE
6 changed files with 153 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
README.md
View File

@ -7,12 +7,7 @@ $ git submodule update --init --recursive
$ yarn install --pure-lockfile
```
Initialize the database:
```terminal
$ sudo -u postgres createuser -P peertube
$ sudo -u postgres createdb -O peertube peertube_search_index
```
The database (Elastic Search) is automatically created by PeerTube on startup.
Then run simultaneously (for example with 2 terminals):

2
server/controllers/api/index.ts
View File

@ -1,8 +1,10 @@
import * as express from 'express'
import { badRequest } from '../../helpers/utils'
import { searchRouter } from './search-videos'
const apiRouter = express.Router()
apiRouter.use('/', searchRouter)
apiRouter.use('/ping', pong)
apiRouter.use('/*', badRequest)

12
server/controllers/api/search-videos.ts
View File

@ -2,15 +2,14 @@ import * as express from 'express'
import { paginationValidator } from '../../middlewares/validators/pagination'
import { setDefaultPagination } from '../../middlewares/pagination'
import { asyncMiddleware } from '../../middlewares/async'
import { getFormattedObjects } from '../../helpers/utils'
import { queryVideos } from '../../lib/elastic-search-videos'
import { formatVideoForAPI, queryVideos } from '../../lib/elastic-search-videos'
import { videosSearchSortValidator } from '../../middlewares/validators/sort'
import { commonVideosFiltersValidator, videosSearchValidator } from '../../middlewares/validators/search'
import { setDefaultSearchSort } from '../../middlewares/sort'
const searchRouter = express.Router()
searchRouter.get('/videos',
searchRouter.get('/search/videos',
paginationValidator,
setDefaultPagination,
videosSearchSortValidator,
@ -27,7 +26,10 @@ export { searchRouter }
// ---------------------------------------------------------------------------
async function searchVideos (req: express.Request, res: express.Response) {
const resultList = await queryVideos(req.body)
const resultList = await queryVideos(req.query)
return res.json(getFormattedObjects(resultList.data, resultList.total))
return res.json({
total: resultList.total,
data: resultList.data.map(v => formatVideoForAPI(v, req.query.fromHost))
})
}

3
server/helpers/custom-validators/search-videos.ts
View File

@ -1,7 +1,8 @@
import { isArray } from './misc'
import { isInt } from 'validator'
function isNumberArray (value: any) {
return isArray(value) && value.every(v => validator.isInt('' + v))
return isArray(value) && value.every(v => isInt('' + v))
}
function isStringArray (value: any) {

168
server/lib/elastic-search-videos.ts
View File

@ -1,11 +1,14 @@
import { CONFIG } from '../initializers/constants'
import { DBVideo, DBVideoDetails, IndexableVideo, IndexableVideoDetails } from '../types/video.model'
import { flatMap } from 'lodash'
import { Avatar } from '@shared/models'
import { Avatar, Video } from '@shared/models'
import { buildSort, elasticSearch } from '../helpers/elastic-search'
import { VideosSearchQuery } from '../types/video-search.model'
import { logger } from '../helpers/logger'
function initVideosIndex () {
logger.info('Initialize %s Elastic Search index.', CONFIG.ELASTIC_SEARCH.INDEXES.VIDEOS)
return elasticSearch.indices.create({
index: CONFIG.ELASTIC_SEARCH.INDEXES.VIDEOS,
body: {
@ -24,25 +27,30 @@ function initVideosIndex () {
})
}
async function indexVideos (videos: IndexableVideo[]) {
async function indexVideos (videos: IndexableVideo[], replace = false) {
const elIdIndex: { [elId: string]: string } = {}
for (const video of videos) {
elIdIndex[video.elasticSearchId] = video.uuid
}
const method = replace ? 'index' : 'update'
const body = flatMap(videos, v => {
const doc = formatVideoForDB(v)
const options = replace
? doc
: { doc, doc_as_upsert: true }
return [
{
update: {
[method]: {
_id: v.elasticSearchId,
_index: CONFIG.ELASTIC_SEARCH.INDEXES.VIDEOS
}
},
{
doc: formatVideo(v),
doc_as_upsert: true
}
options
]
})
@ -51,8 +59,16 @@ async function indexVideos (videos: IndexableVideo[]) {
body
})
const resultBody = result.body
if (resultBody.errors === true) {
const msg = 'Cannot insert data in elastic search.'
logger.error(msg, { err: resultBody })
throw new Error(msg)
}
const created: string[] = result.body.items
.map(i => i.update)
.map(i => i[method])
.filter(i => i.result === 'created')
.map(i => elIdIndex[i._id])
@ -91,7 +107,8 @@ async function queryVideos (search: VideosSearchQuery) {
{
multi_match: {
query: search.search,
fields: [ 'name', 'description' ]
fields: [ 'name', 'description', 'tags' ],
fuzziness: 'AUTO'
}
}
]
@ -138,10 +155,10 @@ async function queryVideos (search: VideosSearchQuery) {
})
}
if (search.nsfw) {
if (search.nsfw && search.nsfw !== 'both') {
filter.push({
term: {
nsfw: search.nsfw
nsfw: (search.nsfw + '') === 'true'
}
})
}
@ -149,7 +166,7 @@ async function queryVideos (search: VideosSearchQuery) {
if (search.categoryOneOf) {
filter.push({
terms: {
category: search.categoryOneOf
'category.id': search.categoryOneOf
}
})
}
@ -157,7 +174,7 @@ async function queryVideos (search: VideosSearchQuery) {
if (search.licenceOneOf) {
filter.push({
terms: {
licence: search.licenceOneOf
'licence.id': search.licenceOneOf
}
})
}
@ -165,7 +182,7 @@ async function queryVideos (search: VideosSearchQuery) {
if (search.languageOneOf) {
filter.push({
terms: {
language: search.languageOneOf
'language.id': search.languageOneOf
}
})
}
@ -210,14 +227,18 @@ async function queryVideos (search: VideosSearchQuery) {
Object.assign(bool, { filter })
const body = {
from: search.start,
size: search.count,
sort: buildVideosSort(search.sort),
query: { bool }
}
logger.debug('Will query Elastic Search for videos.', { body })
const res = await elasticSearch.search({
index: CONFIG.ELASTIC_SEARCH.INDEXES.VIDEOS,
body: {
from: search.start,
size: search.count,
sort: buildVideosSort(search.sort),
query: { bool }
}
body
})
const hits = res.body.hits
@ -229,7 +250,8 @@ export {
queryVideos,
refreshVideosIndex,
initVideosIndex,
listIndexInstances
listIndexInstances,
formatVideoForAPI
}
// ############################################################################
@ -248,7 +270,7 @@ function buildVideosSort (sort: string) {
]
}
function formatVideo (v: IndexableVideo | IndexableVideoDetails): DBVideo | DBVideoDetails {
function formatVideoForDB (v: IndexableVideo | IndexableVideoDetails): DBVideo | DBVideoDetails {
return {
id: v.id,
uuid: v.uuid,
@ -290,7 +312,7 @@ function formatVideo (v: IndexableVideo | IndexableVideoDetails): DBVideo | DBVi
nsfw: v.nsfw,
host: v.host,
tags: (v as any).tags ? (v as any).tags : [],
tags: (v as IndexableVideoDetails).tags ? (v as IndexableVideoDetails).tags : undefined,
account: {
id: v.account.id,
@ -299,7 +321,7 @@ function formatVideo (v: IndexableVideo | IndexableVideoDetails): DBVideo | DBVi
url: v.account.url,
host: v.account.host,
avatar: formatAvatar(v.account)
avatar: formatAvatarForDB(v.account)
},
channel: {
@ -309,12 +331,85 @@ function formatVideo (v: IndexableVideo | IndexableVideoDetails): DBVideo | DBVi
url: v.channel.url,
host: v.channel.host,
avatar: formatAvatar(v.channel)
avatar: formatAvatarForDB(v.channel)
}
}
}
function formatAvatar (obj: { avatar?: Avatar }) {
function formatAvatarForDB (obj: { avatar?: Avatar }) {
if (!obj.avatar) return null
return {
path: obj.avatar.path,
createdAt: obj.avatar.createdAt,
updatedAt: obj.avatar.updatedAt
}
}
function formatVideoForAPI (v: DBVideo, fromHost?: string): Video {
return {
id: v.id,
uuid: v.uuid,
createdAt: new Date(v.createdAt),
updatedAt: new Date(v.updatedAt),
publishedAt: new Date(v.publishedAt),
originallyPublishedAt: v.originallyPublishedAt,
category: {
id: v.category.id,
label: v.category.label
},
licence: {
id: v.licence.id,
label: v.licence.label
},
language: {
id: v.language.id,
label: v.language.label
},
privacy: {
id: v.privacy.id,
label: v.privacy.label
},
name: v.name,
description: v.description,
duration: v.duration,
thumbnailPath: v.thumbnailPath,
previewPath: v.previewPath,
embedPath: v.embedPath,
isLocal: fromHost && fromHost === v.host,
views: v.views,
likes: v.likes,
dislikes: v.dislikes,
nsfw: v.nsfw,
account: {
id: v.account.id,
name: v.account.name,
displayName: v.account.displayName,
url: v.account.url,
host: v.account.host,
avatar: formatAvatarForAPI(v.account)
},
channel: {
id: v.channel.id,
name: v.channel.name,
displayName: v.channel.displayName,
url: v.channel.url,
host: v.channel.host,
avatar: formatAvatarForAPI(v.channel)
}
}
}
function formatAvatarForAPI (obj: { avatar?: Avatar }) {
if (!obj.avatar) return null
return {
@ -354,10 +449,12 @@ function buildChannelOrAccountMapping () {
type: 'keyword'
},
createdAt: {
type: 'date'
type: 'date',
format: 'date_optional_time'
},
updatedAt: {
type: 'date'
type: 'date',
format: 'date_optional_time'
}
}
}
@ -374,19 +471,24 @@ function buildVideosMapping () {
type: 'keyword'
},
createdAt: {
type: 'date'
type: 'date',
format: 'date_optional_time'
},
updatedAt: {
type: 'date'
type: 'date',
format: 'date_optional_time'
},
publishedAt: {
type: 'date'
type: 'date',
format: 'date_optional_time'
},
originallyPublishedAt: {
type: 'date'
type: 'date',
format: 'date_optional_time'
},
indexedAt: {
type: 'date'
type: 'date',
format: 'date_optional_time'
},
category: {

8
server/lib/schedulers/videos-indexer.ts
View File

@ -34,6 +34,10 @@ export class VideosIndexer extends AbstractScheduler {
})
}
scheduleVideoIndexation (host: string, uuid: string) {
this.getVideoQueue.push({ uuid, host })
}
protected async internalExecute () {
return this.runVideosIndexer()
}
@ -61,7 +65,7 @@ export class VideosIndexer extends AbstractScheduler {
// Fetch complete video foreach created video (to get tags)
for (const c of created) {
this.getVideoQueue.push({ uuid: c, host })
this.scheduleVideoIndexation(host, c)
}
}
} while (videos.length === INDEXER_COUNT.VIDEOS && start < 500)
@ -130,7 +134,7 @@ export class VideosIndexer extends AbstractScheduler {
logger.info('Indexing specific video %s of %s.', uuid, host)
await indexVideos([ video ])
await indexVideos([ video ], true)
}
private prepareVideoForDB <T extends Video> (video: T, host: string): T & IndexableDoc {