Generate missing video thumbnails locally

Change-Id: I72b71cf1de2ae2922494cb08c8c9baa607b11562

vector/src/main/java/de/spiritcroc/util/ThumbnailExtractor.kt

@@ -0,0 +1,104 @@
+/*
+ * Copyright 2020 The Matrix.org Foundation C.I.C.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package de.spiritcroc.util
+
+import android.content.Context
+import android.graphics.Bitmap
+import android.media.MediaMetadataRetriever
+import android.net.Uri
+import timber.log.Timber
+import java.io.ByteArrayOutputStream
+import java.io.File
+import java.io.FileOutputStream
+import java.lang.Exception
+import javax.inject.Inject
+
+/**
+ * Based on org.matrix.android.sdk.internal.session.content.ThumbnailExtractor, but more useful for
+ * rendering video thumbnails locally (instead of the SDK's focus on uploading thumbnails).
+ * I.e. we re-use existing MediaData classes, and keep the SDK's class package-private.
+ */
+class ThumbnailExtractor @Inject constructor(
+        private val context: Context
+) {
+
+    companion object {
+        private const val DEBUG_THUMBNAIL_EXTRACTOR = true
+    }
+
+
+    fun extractThumbnail(file: File): Result<File>? {
+        // In case we want to generate thumbnails for non-video files here as well, we need to fix below MIME-type detection.
+        // Currently, it returns false for isMimeTypeVideo on mp4 videos
+        /*
+        val type = MimeTypeMap.getFileExtensionFromUrl(file.absolutePath)
+        if (DEBUG_THUMBNAIL_EXTRACTOR) Timber.v("MIME type is $type: isVideo: ${type.isMimeTypeVideo()}")
+        return if (type.isMimeTypeVideo()) {
+            extractVideoThumbnail(file)
+        } else {
+            null
+        }
+         */
+        // Currently, only video thumbnail generation supported
+        return extractVideoThumbnail(file)
+    }
+
+    private fun getThumbnailCacheFile(videoFile: File): File {
+        val thumbnailCacheDir = File(context.cacheDir, "localThumbnails")
+        return File(thumbnailCacheDir, "${videoFile.canonicalPath}.jpg")
+    }
+
+    private fun extractVideoThumbnail(file: File): Result<File> {
+        val thumbnailFile = getThumbnailCacheFile(file)
+        if (thumbnailFile.exists()) {
+            if (DEBUG_THUMBNAIL_EXTRACTOR) Timber.d("Return cached thumbnail ${thumbnailFile.canonicalPath}")
+            return Result.success(thumbnailFile)
+        }
+        if (DEBUG_THUMBNAIL_EXTRACTOR) Timber.d("Generate thumbnail ${thumbnailFile.canonicalPath}")
+        val mediaMetadataRetriever = MediaMetadataRetriever()
+        try {
+            mediaMetadataRetriever.setDataSource(context, Uri.fromFile(file))
+            mediaMetadataRetriever.frameAtTime?.let { thumbnail ->
+                val outputStream = ByteArrayOutputStream()
+                thumbnail.compress(Bitmap.CompressFormat.JPEG, 80, outputStream)
+                /*
+                val thumbnailWidth = thumbnail.width
+                val thumbnailHeight = thumbnail.height
+                val thumbnailSize = outputStream.size()
+                 */
+                val tmpFile = File(thumbnailFile.parentFile, "${file.name}.part")
+                tmpFile.parentFile?.mkdirs()
+                FileOutputStream(tmpFile).use {
+                    it.write(outputStream.toByteArray())
+                }
+                tmpFile.renameTo(thumbnailFile)
+                thumbnail.recycle()
+                outputStream.reset()
+            } ?: run {
+                Timber.e("Cannot extract video thumbnail at %s", file.canonicalPath)
+                return Result.failure(Exception("Cannot extract video thumbnail at ${file.canonicalPath}"))
+            }
+        } catch (e: Exception) {
+            Timber.e(e, "Cannot extract video thumbnail")
+            return Result.failure(e)
+        } finally {
+            mediaMetadataRetriever.release()
+        }
+        return Result.success(thumbnailFile)
+    }
+
+}

vector/src/main/java/im/vector/app/core/glide/VectorGlideModelLoader.kt

@@ -25,6 +25,7 @@ import com.bumptech.glide.load.model.ModelLoader
 import com.bumptech.glide.load.model.ModelLoaderFactory
 import com.bumptech.glide.load.model.MultiModelLoaderFactory
 import com.bumptech.glide.signature.ObjectKey
+import de.spiritcroc.util.ThumbnailExtractor
 import im.vector.app.core.extensions.singletonEntryPoint
 import im.vector.app.core.files.LocalFilesHelper
 import im.vector.app.features.media.ImageContentRenderer
@@ -71,6 +72,8 @@ class VectorGlideDataFetcher(
     private val localFilesHelper = LocalFilesHelper(context)
     private val activeSessionHolder = context.singletonEntryPoint().activeSessionHolder()
 
+    private val thumbnailExtractor = ThumbnailExtractor(context)
+
     private val client = activeSessionHolder.getSafeActiveSession()?.getOkHttpClient() ?: OkHttpClient()
 
     override fun getDataClass(): Class<InputStream> {
@@ -118,7 +121,7 @@ class VectorGlideDataFetcher(
         }
         // Use the file vector service, will avoid flickering and redownload after upload
         activeSessionHolder.getSafeActiveSession()?.coroutineScope?.launch {
-            val result = runCatching {
+            var result = runCatching {
                 fileService.downloadFile(
                         fileName = data.filename,
                         mimeType = data.mimeType,
@@ -126,6 +129,16 @@ class VectorGlideDataFetcher(
                         elementToDecrypt = data.elementToDecrypt
                 )
             }
+            if (result.isFailure && (data.fallbackUrl != null || data.fallbackElementToDecrypt != null)) {
+                result = runCatching {
+                    fileService.downloadFile(
+                            fileName = data.filename,
+                            mimeType = data.mimeType,
+                            url = data.fallbackUrl,
+                            elementToDecrypt = data.fallbackElementToDecrypt)
+                }
+                result = result.getOrNull()?.let { thumbnailExtractor.extractThumbnail(it) } ?: result
+            }
             withContext(Dispatchers.Main) {
                 result.fold(
                         { callback.onDataReady(it.inputStream()) },

vector/src/main/java/im/vector/app/features/home/room/detail/timeline/factory/MessageItemFactory.kt

@@ -498,7 +498,10 @@ class MessageItemFactory @Inject constructor(
                 maxHeight = maxHeight,
                 width = messageContent.videoInfo?.width,
                 maxWidth = maxWidth,
-                allowNonMxcUrls = informationData.sendState.isSending()
+                allowNonMxcUrls = informationData.sendState.isSending(),
+                // Video fallback for generating thumbnails
+                fallbackUrl = messageContent.getFileUrl(),
+                fallbackElementToDecrypt = messageContent.encryptedFileInfo?.toElementToDecrypt()
         )
 
         val videoData = VideoContentRenderer.Data(

vector/src/main/java/im/vector/app/features/home/room/detail/timeline/image/ImageContentRendererFactory.kt

@@ -57,7 +57,10 @@ fun TimelineEvent.buildImageContentRendererData(maxHeight: Int): ImageContentRen
                             maxHeight = maxHeight,
                             width = videoInfo?.thumbnailInfo?.width,
                             maxWidth = maxHeight * 2,
-                            allowNonMxcUrls = false
+                            allowNonMxcUrls = false,
+                            // Video fallback for generating thumbnails
+                            fallbackUrl = messageVideoContent.getFileUrl(),
+                            fallbackElementToDecrypt = messageVideoContent.encryptedFileInfo?.toElementToDecrypt()
                     )
                 }
         else -> null

vector/src/main/java/im/vector/app/features/media/ImageContentRenderer.kt

@@ -83,7 +83,10 @@ class ImageContentRenderer @Inject constructor(
             val width: Int?,
             val maxWidth: Int,
             // If true will load non mxc url, be careful to set it only for images sent by you
-            override val allowNonMxcUrls: Boolean = false
+            override val allowNonMxcUrls: Boolean = false,
+            // Fallback for videos: generate preview from video
+            val fallbackUrl: String? = null,
+            val fallbackElementToDecrypt: ElementToDecrypt? = null,
     ) : AttachmentData
 
     enum class Mode {
@@ -159,6 +162,7 @@ class ImageContentRenderer @Inject constructor(
         var request = createGlideRequest(data, mode, imageView, size)
                 .listener(object : RequestListener<Drawable> {
                     override fun onLoadFailed(e: GlideException?, model: Any?, target: Target<Drawable>?, isFirstResource: Boolean): Boolean {
+                        Timber.e(e, "Glide image render failed")
                         return false
                     }
 
@@ -268,8 +272,8 @@ class ImageContentRenderer @Inject constructor(
     }
 
     fun createGlideRequest(data: Data, mode: Mode, glideRequests: GlideRequests, size: Size = processSize(data, mode)): GlideRequest<Drawable> {
-        return if (data.elementToDecrypt != null) {
-            // Encrypted image
+        return if (data.elementToDecrypt != null || (data.url == null && data.fallbackUrl != null)) {
+            // Encrypted image, or video without thumbnail url
             glideRequests
                     .load(data)
                     .diskCacheStrategy(DiskCacheStrategy.NONE)