Merge pull request #350 from KryptKode/feat/camera-x-fixes

FIX: properly play media sound on image capture
This commit is contained in:
Tibor Kaputa 2022-10-09 09:17:55 +02:00 committed by GitHub
commit f36f65f08e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 571 additions and 87 deletions

View File

@ -65,7 +65,7 @@ android {
dependencies {
implementation 'com.github.SimpleMobileTools:Simple-Commons:2e9ca234a7'
implementation 'androidx.documentfile:documentfile:1.0.1'
implementation "androidx.exifinterface:exifinterface:1.3.3"
implementation "androidx.exifinterface:exifinterface:1.3.4"
implementation "androidx.lifecycle:lifecycle-runtime-ktx:2.5.1"
implementation 'androidx.window:window:1.1.0-alpha03'

View File

@ -1,11 +0,0 @@
package com.simplemobiletools.camera.extensions
import androidx.camera.core.ImageProxy
// Only for JPEG format
fun ImageProxy.toJpegByteArray(): ByteArray {
val buffer = planes.first().buffer
val jpegImageData = ByteArray(buffer.remaining())
buffer[jpegImageData]
return jpegImageData
}

View File

@ -1,26 +0,0 @@
package com.simplemobiletools.camera.helpers
import android.content.ContentResolver
import android.net.Uri
import androidx.annotation.WorkerThread
import androidx.exifinterface.media.ExifInterface
import com.simplemobiletools.commons.extensions.removeValues
import java.io.IOException
class ExifRemover(private val contentResolver: ContentResolver) {
companion object {
private const val MODE = "rw"
}
@WorkerThread
fun removeExif(uri: Uri) {
try {
val fileDescriptor = contentResolver.openFileDescriptor(uri, MODE)
if (fileDescriptor != null) {
val exifInterface = ExifInterface(fileDescriptor.fileDescriptor)
exifInterface.removeValues()
}
} catch (e: IOException) {
}
}
}

View File

@ -0,0 +1,300 @@
package com.simplemobiletools.camera.helpers
import android.annotation.SuppressLint
import android.content.ContentResolver
import android.content.ContentValues
import android.net.Uri
import android.provider.MediaStore
import androidx.camera.core.ImageCapture
import androidx.camera.core.ImageCapture.Metadata
import androidx.camera.core.ImageCaptureException
import androidx.camera.core.ImageProxy
import androidx.camera.core.internal.compat.workaround.ExifRotationAvailability
import androidx.exifinterface.media.ExifInterface
import com.simplemobiletools.camera.helpers.ImageUtil.CodecFailedException
import com.simplemobiletools.camera.helpers.ImageUtil.imageToJpegByteArray
import com.simplemobiletools.camera.helpers.ImageUtil.jpegImageToJpegByteArray
import com.simplemobiletools.camera.models.MediaOutput
import com.simplemobiletools.commons.extensions.copyTo
import com.simplemobiletools.commons.helpers.ensureBackgroundThread
import com.simplemobiletools.commons.helpers.isQPlus
import java.io.*
import java.util.UUID
/**
* Inspired by
* @see androidx.camera.core.ImageSaver
* */
class ImageSaver private constructor(
private val contentResolver: ContentResolver,
private val image: ImageProxy,
private val mediaOutput: MediaOutput.ImageCaptureOutput,
private val metadata: Metadata,
private val jpegQuality: Int,
private val saveExifAttributes: Boolean,
private val onImageSaved: (Uri) -> Unit,
private val onError: (ImageCaptureException) -> Unit,
) {
companion object {
private const val TEMP_FILE_PREFIX = "SimpleCamera"
private const val TEMP_FILE_SUFFIX = ".tmp"
private const val COPY_BUFFER_SIZE = 1024
private const val PENDING = 1
private const val NOT_PENDING = 0
fun saveImage(
contentResolver: ContentResolver,
image: ImageProxy,
mediaOutput: MediaOutput.ImageCaptureOutput,
metadata: Metadata,
jpegQuality: Int,
saveExifAttributes: Boolean,
onImageSaved: (Uri) -> Unit,
onError: (ImageCaptureException) -> Unit,
) = ImageSaver(
contentResolver = contentResolver,
image = image,
mediaOutput = mediaOutput,
metadata = metadata,
jpegQuality = jpegQuality,
saveExifAttributes = saveExifAttributes,
onImageSaved = onImageSaved,
onError = onError,
).saveImage()
}
fun saveImage() {
ensureBackgroundThread {
// Save the image to a temp file first. This is necessary because ExifInterface only
// supports saving to File.
val tempFile = saveImageToTempFile()
if (tempFile != null) {
copyTempFileToDestination(tempFile)
}
}
}
@SuppressLint("RestrictedApi")
private fun saveImageToTempFile(): File? {
var saveError: SaveError? = null
var errorMessage: String? = null
var exception: Exception? = null
val tempFile = try {
if (mediaOutput is MediaOutput.FileMediaOutput) {
// For saving to file, write to the target folder and rename for better performance.
File(
mediaOutput.file.parent,
TEMP_FILE_PREFIX + UUID.randomUUID().toString() + TEMP_FILE_SUFFIX
)
} else {
File.createTempFile(TEMP_FILE_PREFIX, TEMP_FILE_SUFFIX)
}
} catch (e: IOException) {
postError(SaveError.FILE_IO_FAILED, "Error saving temp file", e)
return null
}
try {
val output = FileOutputStream(tempFile)
val byteArray: ByteArray = imageToJpegByteArray(image, jpegQuality)
output.write(byteArray)
if (saveExifAttributes) {
val exifInterface = ExifInterface(tempFile)
val imageByteArray = jpegImageToJpegByteArray(image)
val inputStream: InputStream = ByteArrayInputStream(imageByteArray)
ExifInterface(inputStream).copyTo(exifInterface)
// Overwrite the original orientation if the quirk exists.
if (!ExifRotationAvailability().shouldUseExifOrientation(image)) {
exifInterface.rotate(image.imageInfo.rotationDegrees)
}
if (metadata.isReversedHorizontal) {
exifInterface.flipHorizontally()
}
if (metadata.isReversedVertical) {
exifInterface.flipVertically()
}
if (metadata.location != null) {
exifInterface.setGpsInfo(metadata.location)
}
exifInterface.saveAttributes()
}
} catch (e: IOException) {
saveError = SaveError.FILE_IO_FAILED
errorMessage = "Failed to write temp file"
exception = e
} catch (e: IllegalArgumentException) {
saveError = SaveError.FILE_IO_FAILED
errorMessage = "Failed to write temp file"
exception = e
} catch (e: CodecFailedException) {
when (e.failureType) {
CodecFailedException.FailureType.ENCODE_FAILED -> {
saveError = SaveError.ENCODE_FAILED
errorMessage = "Failed to encode Image"
}
CodecFailedException.FailureType.DECODE_FAILED -> {
saveError = SaveError.CROP_FAILED
errorMessage = "Failed to crop Image"
}
CodecFailedException.FailureType.UNKNOWN -> {
saveError = SaveError.UNKNOWN
errorMessage = "Failed to transcode Image"
}
}
exception = e
}
if (saveError != null) {
postError(saveError, errorMessage, exception)
tempFile.delete()
return null
}
return tempFile
}
/**
* Copy the temp file to user specified destination.
*
*
* The temp file will be deleted afterwards.
*/
private fun copyTempFileToDestination(tempFile: File) {
var saveError: SaveError? = null
var errorMessage: String? = null
var exception: java.lang.Exception? = null
var outputUri: Uri? = null
try {
when (mediaOutput) {
is MediaOutput.MediaStoreOutput -> {
val values = mediaOutput.contentValues
setContentValuePending(values, PENDING)
outputUri = contentResolver.insert(
mediaOutput.contentUri, values
)
if (outputUri == null) {
saveError = SaveError.FILE_IO_FAILED
errorMessage = "Failed to insert URI."
} else {
if (!copyTempFileToUri(tempFile, outputUri)) {
saveError = SaveError.FILE_IO_FAILED
errorMessage = "Failed to save to URI."
}
setUriNotPending(outputUri)
}
}
is MediaOutput.OutputStreamMediaOutput -> {
copyTempFileToOutputStream(tempFile, mediaOutput.outputStream)
outputUri = mediaOutput.uri
}
is MediaOutput.FileMediaOutput -> {
val targetFile: File = mediaOutput.file
// Normally File#renameTo will overwrite the targetFile even if it already exists.
// Just in case of unexpected behavior on certain platforms or devices, delete the
// target file before renaming.
if (targetFile.exists()) {
targetFile.delete()
}
if (!tempFile.renameTo(targetFile)) {
saveError = SaveError.FILE_IO_FAILED
errorMessage = "Failed to rename file."
}
outputUri = Uri.fromFile(targetFile)
}
MediaOutput.BitmapOutput -> throw UnsupportedOperationException("Bitmap output cannot be saved to disk")
}
} catch (e: IOException) {
saveError = SaveError.FILE_IO_FAILED
errorMessage = "Failed to write destination file."
exception = e
} catch (e: IllegalArgumentException) {
saveError = SaveError.FILE_IO_FAILED
errorMessage = "Failed to write destination file."
exception = e
} finally {
tempFile.delete()
}
outputUri?.let(onImageSaved) ?: postError(saveError!!, errorMessage, exception)
}
private fun postError(saveError: SaveError, errorMessage: String?, exception: Exception?) {
val imageCaptureError = if (saveError == SaveError.FILE_IO_FAILED) {
ImageCapture.ERROR_FILE_IO
} else {
ImageCapture.ERROR_UNKNOWN
}
onError.invoke(ImageCaptureException(imageCaptureError, errorMessage!!, exception!!))
}
/**
* Removes IS_PENDING flag during the writing to [Uri].
*/
private fun setUriNotPending(outputUri: Uri) {
if (isQPlus()) {
val values = ContentValues()
setContentValuePending(values, NOT_PENDING)
contentResolver.update(outputUri, values, null, null)
}
}
/** Set IS_PENDING flag to [ContentValues]. */
private fun setContentValuePending(values: ContentValues, isPending: Int) {
if (isQPlus()) {
values.put(MediaStore.Images.Media.IS_PENDING, isPending)
}
}
/**
* Copies temp file to [Uri].
*
* @return false if the [Uri] is not writable.
*/
@Throws(IOException::class)
private fun copyTempFileToUri(tempFile: File, uri: Uri): Boolean {
contentResolver.openOutputStream(uri).use { outputStream ->
if (outputStream == null) {
// The URI is not writable.
return false
}
copyTempFileToOutputStream(tempFile, outputStream)
}
return true
}
@Throws(IOException::class)
private fun copyTempFileToOutputStream(
tempFile: File, outputStream: OutputStream
) {
FileInputStream(tempFile).use { inputStream ->
val buf = ByteArray(COPY_BUFFER_SIZE)
var len: Int
while (inputStream.read(buf).also { len = it } > 0) {
outputStream.write(buf, 0, len)
}
}
}
/** Type of error that occurred during save */
enum class SaveError {
/** Failed to write to or close the file */
FILE_IO_FAILED,
/** Failure when attempting to encode image */
ENCODE_FAILED,
/** Failure when attempting to crop image */
CROP_FAILED, UNKNOWN
}
}

View File

@ -0,0 +1,231 @@
package com.simplemobiletools.camera.helpers
import android.graphics.*
import androidx.annotation.IntRange
import androidx.camera.core.ImageProxy
import java.io.ByteArrayOutputStream
import java.io.IOException
/**
* Utility class for image related operations.
* @see androidx.camera.core.internal.utils.ImageUtil
*/
object ImageUtil {
@Throws(CodecFailedException::class)
fun imageToJpegByteArray(image: ImageProxy, jpegQuality: Int): ByteArray {
val shouldCropImage = shouldCropImage(image)
return when (image.format) {
ImageFormat.JPEG -> {
if (!shouldCropImage) {
// When cropping is unnecessary, the byte array doesn't need to be decoded and
// re-encoded again. Therefore, jpegQuality is unnecessary in this case.
jpegImageToJpegByteArray(image)
} else {
jpegImageToJpegByteArray(image, image.cropRect, jpegQuality)
}
}
ImageFormat.YUV_420_888 -> {
yuvImageToJpegByteArray(image, if (shouldCropImage) image.cropRect else null, jpegQuality)
}
else -> {
// Unrecognized image format
byteArrayOf()
}
}
}
/**
* Converts JPEG [ImageProxy] to JPEG byte array.
*/
fun jpegImageToJpegByteArray(image: ImageProxy): ByteArray {
require(image.format == ImageFormat.JPEG) { "Incorrect image format of the input image proxy: " + image.format }
val planes = image.planes
val buffer = planes[0].buffer
val data = ByteArray(buffer.capacity())
buffer.rewind()
buffer[data]
return data
}
/**
* Converts JPEG [ImageProxy] to JPEG byte array. The input JPEG image will be cropped
* by the specified crop rectangle and compressed by the specified quality value.
*/
@Throws(CodecFailedException::class)
private fun jpegImageToJpegByteArray(
image: ImageProxy,
cropRect: Rect,
@IntRange(from = 1, to = 100) jpegQuality: Int,
): ByteArray {
require(image.format == ImageFormat.JPEG) { "Incorrect image format of the input image proxy: " + image.format }
var data = jpegImageToJpegByteArray(image)
data = cropJpegByteArray(data, cropRect, jpegQuality)
return data
}
/**
* Converts YUV_420_888 [ImageProxy] to JPEG byte array. The input YUV_420_888 image
* will be cropped if a non-null crop rectangle is specified. The output JPEG byte array will
* be compressed by the specified quality value.
*/
@Throws(CodecFailedException::class)
private fun yuvImageToJpegByteArray(
image: ImageProxy,
cropRect: Rect?,
@IntRange(from = 1, to = 100) jpegQuality: Int,
): ByteArray {
require(image.format == ImageFormat.YUV_420_888) { "Incorrect image format of the input image proxy: " + image.format }
return nv21ToJpeg(
yuv_420_888toNv21(image),
image.width,
image.height,
cropRect,
jpegQuality
)
}
/**
* [android.media.Image] to NV21 byte array.
*/
private fun yuv_420_888toNv21(image: ImageProxy): ByteArray {
val yPlane = image.planes[0]
val uPlane = image.planes[1]
val vPlane = image.planes[2]
val yBuffer = yPlane.buffer
val uBuffer = uPlane.buffer
val vBuffer = vPlane.buffer
yBuffer.rewind()
uBuffer.rewind()
vBuffer.rewind()
val ySize = yBuffer.remaining()
var position = 0
val nv21 = ByteArray(ySize + image.width * image.height / 2)
// Add the full y buffer to the array. If rowStride > 1, some padding may be skipped.
for (row in 0 until image.height) {
yBuffer[nv21, position, image.width]
position += image.width
yBuffer.position(
Math.min(ySize, yBuffer.position() - image.width + yPlane.rowStride)
)
}
val chromaHeight = image.height / 2
val chromaWidth = image.width / 2
val vRowStride = vPlane.rowStride
val uRowStride = uPlane.rowStride
val vPixelStride = vPlane.pixelStride
val uPixelStride = uPlane.pixelStride
// Interleave the u and v frames, filling up the rest of the buffer. Use two line buffers to
// perform faster bulk gets from the byte buffers.
val vLineBuffer = ByteArray(vRowStride)
val uLineBuffer = ByteArray(uRowStride)
for (row in 0 until chromaHeight) {
vBuffer[vLineBuffer, 0, Math.min(vRowStride, vBuffer.remaining())]
uBuffer[uLineBuffer, 0, Math.min(uRowStride, uBuffer.remaining())]
var vLineBufferPosition = 0
var uLineBufferPosition = 0
for (col in 0 until chromaWidth) {
nv21[position++] = vLineBuffer[vLineBufferPosition]
nv21[position++] = uLineBuffer[uLineBufferPosition]
vLineBufferPosition += vPixelStride
uLineBufferPosition += uPixelStride
}
}
return nv21
}
/**
* Crops JPEG byte array with given [android.graphics.Rect].
*/
@Throws(CodecFailedException::class)
private fun cropJpegByteArray(
data: ByteArray,
cropRect: Rect,
@IntRange(from = 1, to = 100) jpegQuality: Int,
): ByteArray {
val bitmap: Bitmap
try {
val decoder = BitmapRegionDecoder.newInstance(
data, 0, data.size,
false
)
bitmap = decoder.decodeRegion(cropRect, BitmapFactory.Options())
decoder.recycle()
} catch (e: IllegalArgumentException) {
throw CodecFailedException(
"Decode byte array failed with illegal argument.$e",
CodecFailedException.FailureType.DECODE_FAILED
)
} catch (e: IOException) {
throw CodecFailedException(
"Decode byte array failed.",
CodecFailedException.FailureType.DECODE_FAILED
)
}
val out = ByteArrayOutputStream()
val success = bitmap.compress(Bitmap.CompressFormat.JPEG, jpegQuality, out)
if (!success) {
throw CodecFailedException(
"Encode bitmap failed.",
CodecFailedException.FailureType.ENCODE_FAILED
)
}
bitmap.recycle()
return out.toByteArray()
}
@Throws(CodecFailedException::class)
private fun nv21ToJpeg(
nv21: ByteArray,
width: Int,
height: Int,
cropRect: Rect?,
@IntRange(from = 1, to = 100) jpegQuality: Int,
): ByteArray {
val out = ByteArrayOutputStream()
val yuv = YuvImage(nv21, ImageFormat.NV21, width, height, null)
val success = yuv.compressToJpeg(
cropRect ?: Rect(0, 0, width, height),
jpegQuality, out
)
if (!success) {
throw CodecFailedException(
"YuvImage failed to encode jpeg.",
CodecFailedException.FailureType.ENCODE_FAILED
)
}
return out.toByteArray()
}
/**
* Checks whether the image's crop rectangle is the same as the source image size.
*/
private fun shouldCropImage(image: ImageProxy): Boolean {
return shouldCropImage(
image.width, image.height, image.cropRect.width(),
image.cropRect.height()
)
}
/**
* Checks whether the image's crop rectangle is the same as the source image size.
*/
private fun shouldCropImage(
sourceWidth: Int, sourceHeight: Int, cropRectWidth: Int,
cropRectHeight: Int
): Boolean {
return sourceWidth != cropRectWidth || sourceHeight != cropRectHeight
}
/**
* Exception for error during transcoding image.
*/
class CodecFailedException internal constructor(message: String, val failureType: FailureType) : Exception(message) {
enum class FailureType {
ENCODE_FAILED, DECODE_FAILED, UNKNOWN
}
}
}

View File

@ -33,7 +33,7 @@ class MediaOutputHelper(
private val mediaStorageDir = activity.config.savePhotosFolder
private val contentResolver = activity.contentResolver
fun getImageMediaOutput(): MediaOutput {
fun getImageMediaOutput(): MediaOutput.ImageCaptureOutput {
return try {
if (is3rdPartyIntent) {
if (outputUri != null) {
@ -56,7 +56,7 @@ class MediaOutputHelper(
}
}
fun getVideoMediaOutput(): MediaOutput {
fun getVideoMediaOutput(): MediaOutput.VideoCaptureOutput {
return try {
if (is3rdPartyIntent) {
if (outputUri != null) {

View File

@ -57,7 +57,6 @@ class CameraXPreview(
private val windowMetricsCalculator = WindowMetricsCalculator.getOrCreate()
private val videoQualityManager = VideoQualityManager(activity)
private val imageQualityManager = ImageQualityManager(activity)
private val exifRemover = ExifRemover(contentResolver)
private val mediaSizeStore = MediaSizeStore(config)
private val orientationEventListener = object : OrientationEventListener(activity, SensorManager.SENSOR_DELAY_NORMAL) {
@ -422,12 +421,14 @@ class CameraXPreview(
}
val mediaOutput = mediaOutputHelper.getImageMediaOutput()
if (mediaOutput is MediaOutput.BitmapOutput) {
imageCapture.takePicture(mainExecutor, object : OnImageCapturedCallback() {
override fun onCaptureSuccess(image: ImageProxy) {
ensureBackgroundThread {
image.use {
val bitmap = BitmapUtils.makeBitmap(image.toJpegByteArray())
imageCapture.takePicture(mainExecutor, object : OnImageCapturedCallback() {
override fun onCaptureSuccess(image: ImageProxy) {
playShutterSoundIfEnabled()
ensureBackgroundThread {
image.use {
if (mediaOutput is MediaOutput.BitmapOutput) {
val imageBytes = ImageUtil.jpegImageToJpegByteArray(image)
val bitmap = BitmapUtils.makeBitmap(imageBytes)
activity.runOnUiThread {
listener.toggleBottomButtons(false)
if (bitmap != null) {
@ -436,44 +437,31 @@ class CameraXPreview(
cameraErrorHandler.handleImageCaptureError(ERROR_CAPTURE_FAILED)
}
}
} else {
ImageSaver.saveImage(
contentResolver = contentResolver,
image = image,
mediaOutput = mediaOutput,
metadata = metadata,
jpegQuality = config.photoQuality,
saveExifAttributes = config.savePhotoMetadata,
onImageSaved = { savedUri ->
activity.runOnUiThread {
listener.toggleBottomButtons(false)
listener.onMediaSaved(savedUri)
}
},
onError = ::handleImageCaptureError
)
}
}
}
override fun onError(exception: ImageCaptureException) {
handleImageCaptureError(exception)
}
})
} else {
val outputOptionsBuilder = when (mediaOutput) {
is MediaOutput.MediaStoreOutput -> OutputFileOptions.Builder(contentResolver, mediaOutput.contentUri, mediaOutput.contentValues)
is MediaOutput.OutputStreamMediaOutput -> OutputFileOptions.Builder(mediaOutput.outputStream)
else -> throw IllegalArgumentException("Unexpected option for image ")
}
val outputOptions = outputOptionsBuilder.setMetadata(metadata).build()
imageCapture.takePicture(outputOptions, mainExecutor, object : OnImageSavedCallback {
override fun onImageSaved(outputFileResults: OutputFileResults) {
ensureBackgroundThread {
val savedUri = mediaOutput.uri ?: outputFileResults.savedUri!!
if (!config.savePhotoMetadata) {
exifRemover.removeExif(savedUri)
}
activity.runOnUiThread {
listener.toggleBottomButtons(false)
listener.onMediaSaved(savedUri)
}
}
}
override fun onError(exception: ImageCaptureException) {
handleImageCaptureError(exception)
}
})
}
playShutterSoundIfEnabled()
override fun onError(exception: ImageCaptureException) {
handleImageCaptureError(exception)
}
})
}
private fun handleImageCaptureError(exception: ImageCaptureException) {
@ -518,7 +506,6 @@ class CameraXPreview(
MediaStoreOutputOptions.Builder(contentResolver, mediaOutput.contentUri).setContentValues(mediaOutput.contentValues).build()
.let { videoCapture.output.prepareRecording(activity, it) }
}
else -> throw IllegalArgumentException("Unexpected output option for video $mediaOutput")
}
currentRecording = recording.withAudioEnabled()
@ -540,7 +527,7 @@ class CameraXPreview(
if (recordEvent.hasError()) {
cameraErrorHandler.handleVideoRecordingError(recordEvent.error)
} else {
listener.onMediaSaved(mediaOutput.uri ?: recordEvent.outputResults.outputUri)
listener.onMediaSaved(recordEvent.outputResults.outputUri)
}
}
}

View File

@ -9,25 +9,28 @@ import java.io.OutputStream
sealed class MediaOutput(
open val uri: Uri?,
) {
sealed interface ImageCaptureOutput
sealed interface VideoCaptureOutput
data class MediaStoreOutput(
val contentValues: ContentValues,
val contentUri: Uri,
) : MediaOutput(null)
) : MediaOutput(null), ImageCaptureOutput, VideoCaptureOutput
data class OutputStreamMediaOutput(
val outputStream: OutputStream,
override val uri: Uri,
) : MediaOutput(uri)
) : MediaOutput(uri), ImageCaptureOutput
data class FileDescriptorMediaOutput(
val fileDescriptor: ParcelFileDescriptor,
override val uri: Uri,
) : MediaOutput(uri)
) : MediaOutput(uri), VideoCaptureOutput
data class FileMediaOutput(
val file: File,
override val uri: Uri,
) : MediaOutput(uri)
) : MediaOutput(uri), VideoCaptureOutput, ImageCaptureOutput
object BitmapOutput : MediaOutput(null)
object BitmapOutput : MediaOutput(null), ImageCaptureOutput
}