diff --git a/app/src/main/java/jp/juggler/subwaytooter/util/AttachmentRequest.kt b/app/src/main/java/jp/juggler/subwaytooter/util/AttachmentRequest.kt index 98158c78..608d5e05 100644 --- a/app/src/main/java/jp/juggler/subwaytooter/util/AttachmentRequest.kt +++ b/app/src/main/java/jp/juggler/subwaytooter/util/AttachmentRequest.kt @@ -3,6 +3,8 @@ package jp.juggler.subwaytooter.util import android.content.Context import android.graphics.Bitmap import android.net.Uri +import jp.juggler.media.generateTempFile +import jp.juggler.media.transcodeAudio import jp.juggler.subwaytooter.R import jp.juggler.subwaytooter.api.entity.InstanceType import jp.juggler.subwaytooter.api.entity.TootInstance @@ -32,14 +34,26 @@ class AttachmentRequest( ) { companion object { private val log = LogCategory("AttachmentRequest") - } - fun hasServerSupport(mimeType: String) = - mediaConfig?.jsonArray("supported_mime_types")?.contains(mimeType) - ?: when (instance.instanceType) { - InstanceType.Pixelfed -> AttachmentUploader.acceptableMimeTypesPixelfed - else -> AttachmentUploader.acceptableMimeTypes - }.contains(mimeType) + private val goodAudioType = setOf( + "audio/flac", + "audio/mp3", + "audio/ogg", + "audio/vnd.wave", + "audio/vorbis", + "audio/wav", + "audio/wave", + "audio/webm", + "audio/x-pn-wave", + "audio/x-wav", + "audio/3gpp", + ) +// val badAudioType = setOf( +// "audio/mpeg","audio/aac", +// "audio/m4a","audio/x-m4a","audio/mp4", +// "video/x-ms-asf", +// ) + } suspend fun createOpener(): InputStreamOpener { @@ -122,39 +136,50 @@ class AttachmentRequest( ) } + private fun hasServerSupport(mimeType: String) = + mediaConfig?.jsonArray("supported_mime_types")?.contains(mimeType) + ?: when (instance.instanceType) { + InstanceType.Pixelfed -> AttachmentUploader.acceptableMimeTypesPixelfed + else -> AttachmentUploader.acceptableMimeTypes + }.contains(mimeType) + private fun createResizedImageOpener( forcePng: Boolean = false, ): InputStreamOpener { - val cacheDir = context.externalCacheDir - ?.apply { mkdirs() } - ?: error("getExternalCacheDir returns null.") - - val outputMimeType = if (forcePng || mimeType == AttachmentUploader.MIME_TYPE_PNG) { - AttachmentUploader.MIME_TYPE_PNG - } else { - AttachmentUploader.MIME_TYPE_JPEG - } - - val tempFile = File(cacheDir, "tmp." + Thread.currentThread().id) - val bitmap = createResizedBitmap( - context, - uri, - imageResizeConfig, - skipIfNoNeedToResizeAndRotate = !forcePng, - serverMaxSqPixel = serverMaxSqPixel - ) ?: error("createResizedBitmap returns null.") - pa.progress = context.getString(R.string.attachment_handling_compress) + val tempFile = context.generateTempFile("createResizedImageOpener") try { - FileOutputStream(tempFile).use { os -> - if (outputMimeType == AttachmentUploader.MIME_TYPE_PNG) { - bitmap.compress(Bitmap.CompressFormat.PNG, 100, os) - } else { - bitmap.compress(Bitmap.CompressFormat.JPEG, 95, os) + pa.progress = context.getString(R.string.attachment_handling_compress) + + val bitmap = createResizedBitmap( + context, + uri, + imageResizeConfig, + skipIfNoNeedToResizeAndRotate = !forcePng, + serverMaxSqPixel = serverMaxSqPixel + ) ?: error("createResizedBitmap returns null.") + try { + val outputMimeType = when { + forcePng || mimeType == AttachmentUploader.MIME_TYPE_PNG -> + AttachmentUploader.MIME_TYPE_PNG + + else -> AttachmentUploader.MIME_TYPE_JPEG } + FileOutputStream(tempFile).use { outStream -> + when (outputMimeType) { + AttachmentUploader.MIME_TYPE_PNG -> + bitmap.compress(Bitmap.CompressFormat.PNG, 100, outStream) + + else -> + bitmap.compress(Bitmap.CompressFormat.JPEG, 95, outStream) + } + } + return tempFileOpener(tempFile, outputMimeType, isImage = true) + } finally { + bitmap.recycle() } - return tempFileOpener(tempFile, outputMimeType, isImage = true) - } finally { - bitmap.recycle() + } catch (ex: Throwable) { + tempFile.delete() + throw ex } } @@ -168,13 +193,13 @@ class AttachmentRequest( val outFile = File(cacheDir, "movie." + Thread.currentThread().id + ".mp4") var resultFile: File? = null - // 入力ファイルをコピーする - (context.contentResolver.openInputStream(uri) - ?: error("openInputStream returns null.")).use { inStream -> - FileOutputStream(tempFile).use { inStream.copyTo(it) } - } - try { + // 入力ファイルをコピーする + (context.contentResolver.openInputStream(uri) + ?: error("openInputStream returns null.")).use { inStream -> + FileOutputStream(tempFile).use { inStream.copyTo(it) } + } + // 動画のメタデータを調べる val info = tempFile.videoInfo @@ -224,27 +249,34 @@ class AttachmentRequest( } } - private val aacMimeTypes = listOf( - "audio/aac", "audio/aacp", "audio/3gpp", "audio/3gpp2", - "audio/mp4", "audio/MP4A-LATM", "audio/mpeg4-generic" - ).map { it.lowercase() }.toSet() - - private fun createResizedAudioOpener( - originalBytes: Long, - ): InputStreamOpener { - if (hasServerSupport("audio/aac") && aacMimeTypes.contains(mimeType.lowercase())) { - mimeType = "audio/aac" - } - - // サーバ側がサポートしてる形式でサイズ以内なら - if (hasServerSupport(mimeType) && originalBytes <= maxBytesVideo) { - return contentUriOpener( + private suspend fun createResizedAudioOpener(srcBytes: Long): InputStreamOpener = + when { + hasServerSupport(mimeType) && + goodAudioType.contains(mimeType) && + srcBytes <= maxBytesVideo -> contentUriOpener( context.contentResolver, uri, mimeType, isImage = false, ) + + else -> { + pa.progress = context.getString(R.string.attachment_handling_compress) + val (tempFile, outMimeType) = transcodeAudio( + context, + uri, + mimeType + ) + // このワークアラウンドはうまくいかなかった +// outMimeType = when (outMimeType) { +// "audio/mp4" -> "audio/x-m4a" +// else -> outMimeType +// } + tempFileOpener( + tempFile, + outMimeType, + isImage = false, + ) + } } - error("audio conversion is not yet supported.") - } } diff --git a/app/src/main/java/jp/juggler/subwaytooter/util/AttachmentUploader.kt b/app/src/main/java/jp/juggler/subwaytooter/util/AttachmentUploader.kt index ec83f245..b844bbd5 100644 --- a/app/src/main/java/jp/juggler/subwaytooter/util/AttachmentUploader.kt +++ b/app/src/main/java/jp/juggler/subwaytooter/util/AttachmentUploader.kt @@ -6,6 +6,7 @@ import android.os.Handler import android.os.SystemClock import androidx.annotation.WorkerThread import androidx.appcompat.app.AppCompatActivity +import androidx.media3.common.MimeTypes import jp.juggler.subwaytooter.R import jp.juggler.subwaytooter.api.TootApiCallback import jp.juggler.subwaytooter.api.TootApiClient @@ -295,15 +296,15 @@ class AttachmentUploader( val isAccepteble = instance.configuration ?.jsonObject("media_attachments") ?.jsonArray("supported_mime_types") - ?.contains(mimeType) + ?.contains(opener.mimeType) ?: when (instance.instanceType) { InstanceType.Pixelfed -> acceptableMimeTypesPixelfed else -> acceptableMimeTypes - }.contains(mimeType) + }.contains(opener.mimeType) if (!isAccepteble) { return TootApiResult( - safeContext.getString(R.string.mime_type_not_acceptable, mimeType) + safeContext.getString(R.string.mime_type_not_acceptable, opener.mimeType) ) } diff --git a/base/build.gradle b/base/build.gradle index 709ef386..3a9f324b 100644 --- a/base/build.gradle +++ b/base/build.gradle @@ -98,7 +98,9 @@ dependencies { api "org.jetbrains.kotlinx:kotlinx-datetime:0.4.0" api "org.jetbrains.kotlinx:kotlinx-serialization-json:1.4.1" api "ru.gildor.coroutines:kotlin-coroutines-okhttp:1.0" - + api "androidx.media3:media3-transformer:1.0.1" + api "androidx.media3:media3-effect:1.0.1" + api "androidx.media3:media3-common:1.0.1" // commons-codecをapiにすると、端末上の古いjarが使われてしまう // declaration of 'org.apache.commons.codec.binary.Base64' appears in /system/framework/org.apache.http.legacy.jar) androidTestImplementation "commons-codec:commons-codec:1.15" diff --git a/base/src/main/java/jp/juggler/media/AudioTranscoder.kt b/base/src/main/java/jp/juggler/media/AudioTranscoder.kt new file mode 100644 index 00000000..4a4f16e6 --- /dev/null +++ b/base/src/main/java/jp/juggler/media/AudioTranscoder.kt @@ -0,0 +1,112 @@ +package jp.juggler.media + +import android.content.Context +import android.net.Uri +import android.os.Looper +import androidx.media3.common.MediaItem +import androidx.media3.common.MimeTypes +import androidx.media3.transformer.TransformationException +import androidx.media3.transformer.TransformationRequest +import androidx.media3.transformer.TransformationResult +import androidx.media3.transformer.Transformer +import jp.juggler.util.log.LogCategory +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.suspendCancellableCoroutine +import kotlinx.coroutines.withContext +import java.io.File +import java.io.FileOutputStream +import java.nio.file.Files +import java.nio.file.Paths +import java.util.UUID +import kotlin.coroutines.resumeWithException + +private val log = LogCategory("transcodeAudio") + +val generateTempFileLock = Any() + +fun Context.generateTempFile(prefix: String) = + synchronized(generateTempFileLock) { + val cacheDir = externalCacheDir ?: cacheDir ?: error("missing cacheDir") + cacheDir.mkdirs() + val path = Paths.get(cacheDir.canonicalPath) + if (!Files.isDirectory(path)) error("cacheDir is not directory. $cacheDir") + if (!Files.isWritable(path)) error("cacheDir is not writable. $cacheDir") + // 重複しない一時ファイル名を探す + var tempFile: File + do { + tempFile = File(cacheDir, "$prefix-${UUID.randomUUID()}") + } while (tempFile.exists()) + // すぐにファイルを作成する + FileOutputStream(tempFile).use {} + tempFile + } + +@OptIn(ExperimentalCoroutinesApi::class) +@androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) +suspend fun transcodeAudio( + context: Context, + inUri: Uri, + inMimeType: String, +): Pair { + val inputMediaItem = MediaItem.fromUri(inUri) + // トランスコードに指定するmimeType + val encodeMimeType = MimeTypes.AUDIO_AAC + // MediaStore登録に使うmimeType。 audio/mp4a-latm だと失敗するのホント困る + val storeMimeType = "audio/mp4" + val tmpFile = context.generateTempFile("transcodeAudio") + + // Transformerは単一スレッドで処理する要件 + val result: TransformationResult = withContext(Dispatchers.Main.immediate) { + val looper = Looper.getMainLooper() + suspendCancellableCoroutine { cont -> + val transformerListener = object : Transformer.Listener { + override fun onTransformationCompleted( + inputMediaItem: MediaItem, + transformationResult: TransformationResult, + ) { + log.i("onTransformationCompleted inputMediaItem=$inputMediaItem transformationResult=$transformationResult") + if (cont.isActive) cont.resume(transformationResult) {} + } + + override fun onTransformationError( + inputMediaItem: MediaItem, + exception: TransformationException, + ) { + log.e( + exception, + "onTransformationError inputMediaItem=$inputMediaItem" + ) + if (cont.isActive) cont.resumeWithException(exception) + } + + override fun onFallbackApplied( + inputMediaItem: MediaItem, + originalTransformationRequest: TransformationRequest, + fallbackTransformationRequest: TransformationRequest, + ) { + log.i("onFallbackApplied inputMediaItem=$inputMediaItem original=$originalTransformationRequest fallback=$fallbackTransformationRequest") + } + } + val transformer = Transformer.Builder(context) + .setLooper(looper) + .setTransformationRequest( + TransformationRequest.Builder() + .setAudioMimeType(encodeMimeType) + .build() + ) + .setRemoveVideo(true) + .setRemoveAudio(false) + .addListener(transformerListener) + .build() + transformer.startTransformation(inputMediaItem, tmpFile.canonicalPath) + cont.invokeOnCancellation { + transformer.cancel() + } + } + } + result.run { + log.i("result: durationMs=$durationMs, fileSizeBytes=$fileSizeBytes, averageAudioBitrate=$averageAudioBitrate, averageVideoBitrate=$averageVideoBitrate, videoFrameCount=$videoFrameCount") + } + return Pair(tmpFile, storeMimeType) +}