Mastodon
/
SubwayTooter-Android-App
mirror of https://github.com/tateisu/SubwayTooter
1
0
Fork 0
Code Issues Releases Wiki Activity

オーディオのトランスコード処理を追加。しかしサーバ側問題でうまくない…

This commit is contained in:
tateisu 2023-04-30 18:08:52 +09:00
parent 58b14554a8
commit c5f262112e
4 changed files with 208 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

146
app/src/main/java/jp/juggler/subwaytooter/util/AttachmentRequest.kt
View File

@ -3,6 +3,8 @@ package jp.juggler.subwaytooter.util
import android.content.Context
import android.graphics.Bitmap
import android.net.Uri
import jp.juggler.media.generateTempFile
import jp.juggler.media.transcodeAudio
import jp.juggler.subwaytooter.R
import jp.juggler.subwaytooter.api.entity.InstanceType
import jp.juggler.subwaytooter.api.entity.TootInstance
@ -32,14 +34,26 @@ class AttachmentRequest(
) {
companion object {
private val log = LogCategory("AttachmentRequest")
}
fun hasServerSupport(mimeType: String) =
mediaConfig?.jsonArray("supported_mime_types")?.contains(mimeType)
?: when (instance.instanceType) {
InstanceType.Pixelfed -> AttachmentUploader.acceptableMimeTypesPixelfed
else -> AttachmentUploader.acceptableMimeTypes
}.contains(mimeType)
private val goodAudioType = setOf(
"audio/flac",
"audio/mp3",
"audio/ogg",
"audio/vnd.wave",
"audio/vorbis",
"audio/wav",
"audio/wave",
"audio/webm",
"audio/x-pn-wave",
"audio/x-wav",
"audio/3gpp",
)
// val badAudioType = setOf(
// "audio/mpeg","audio/aac",
// "audio/m4a","audio/x-m4a","audio/mp4",
// "video/x-ms-asf",
// )
}
suspend fun createOpener(): InputStreamOpener {
@ -122,39 +136,50 @@ class AttachmentRequest(
)
}
private fun hasServerSupport(mimeType: String) =
mediaConfig?.jsonArray("supported_mime_types")?.contains(mimeType)
?: when (instance.instanceType) {
InstanceType.Pixelfed -> AttachmentUploader.acceptableMimeTypesPixelfed
else -> AttachmentUploader.acceptableMimeTypes
}.contains(mimeType)
private fun createResizedImageOpener(
forcePng: Boolean = false,
): InputStreamOpener {
val cacheDir = context.externalCacheDir
?.apply { mkdirs() }
?: error("getExternalCacheDir returns null.")
val outputMimeType = if (forcePng || mimeType == AttachmentUploader.MIME_TYPE_PNG) {
AttachmentUploader.MIME_TYPE_PNG
} else {
AttachmentUploader.MIME_TYPE_JPEG
}
val tempFile = File(cacheDir, "tmp." + Thread.currentThread().id)
val bitmap = createResizedBitmap(
context,
uri,
imageResizeConfig,
skipIfNoNeedToResizeAndRotate = !forcePng,
serverMaxSqPixel = serverMaxSqPixel
) ?: error("createResizedBitmap returns null.")
pa.progress = context.getString(R.string.attachment_handling_compress)
val tempFile = context.generateTempFile("createResizedImageOpener")
try {
FileOutputStream(tempFile).use { os ->
if (outputMimeType == AttachmentUploader.MIME_TYPE_PNG) {
bitmap.compress(Bitmap.CompressFormat.PNG, 100, os)
} else {
bitmap.compress(Bitmap.CompressFormat.JPEG, 95, os)
pa.progress = context.getString(R.string.attachment_handling_compress)
val bitmap = createResizedBitmap(
context,
uri,
imageResizeConfig,
skipIfNoNeedToResizeAndRotate = !forcePng,
serverMaxSqPixel = serverMaxSqPixel
) ?: error("createResizedBitmap returns null.")
try {
val outputMimeType = when {
forcePng || mimeType == AttachmentUploader.MIME_TYPE_PNG ->
AttachmentUploader.MIME_TYPE_PNG
else -> AttachmentUploader.MIME_TYPE_JPEG
}
FileOutputStream(tempFile).use { outStream ->
when (outputMimeType) {
AttachmentUploader.MIME_TYPE_PNG ->
bitmap.compress(Bitmap.CompressFormat.PNG, 100, outStream)
else ->
bitmap.compress(Bitmap.CompressFormat.JPEG, 95, outStream)
}
}
return tempFileOpener(tempFile, outputMimeType, isImage = true)
} finally {
bitmap.recycle()
}
return tempFileOpener(tempFile, outputMimeType, isImage = true)
} finally {
bitmap.recycle()
} catch (ex: Throwable) {
tempFile.delete()
throw ex
}
}
@ -168,13 +193,13 @@ class AttachmentRequest(
val outFile = File(cacheDir, "movie." + Thread.currentThread().id + ".mp4")
var resultFile: File? = null
// 入力ファイルをコピーする
(context.contentResolver.openInputStream(uri)
?: error("openInputStream returns null.")).use { inStream ->
FileOutputStream(tempFile).use { inStream.copyTo(it) }
}
try {
// 入力ファイルをコピーする
(context.contentResolver.openInputStream(uri)
?: error("openInputStream returns null.")).use { inStream ->
FileOutputStream(tempFile).use { inStream.copyTo(it) }
}
// 動画のメタデータを調べる
val info = tempFile.videoInfo
@ -224,27 +249,34 @@ class AttachmentRequest(
}
}
private val aacMimeTypes = listOf(
"audio/aac", "audio/aacp", "audio/3gpp", "audio/3gpp2",
"audio/mp4", "audio/MP4A-LATM", "audio/mpeg4-generic"
).map { it.lowercase() }.toSet()
private fun createResizedAudioOpener(
originalBytes: Long,
): InputStreamOpener {
if (hasServerSupport("audio/aac") && aacMimeTypes.contains(mimeType.lowercase())) {
mimeType = "audio/aac"
}
// サーバ側がサポートしてる形式でサイズ以内なら
if (hasServerSupport(mimeType) && originalBytes <= maxBytesVideo) {
return contentUriOpener(
private suspend fun createResizedAudioOpener(srcBytes: Long): InputStreamOpener =
when {
hasServerSupport(mimeType) &&
goodAudioType.contains(mimeType) &&
srcBytes <= maxBytesVideo -> contentUriOpener(
context.contentResolver,
uri,
mimeType,
isImage = false,
)
else -> {
pa.progress = context.getString(R.string.attachment_handling_compress)
val (tempFile, outMimeType) = transcodeAudio(
context,
uri,
mimeType
)
// このワークアラウンドはうまくいかなかった
// outMimeType = when (outMimeType) {
// "audio/mp4" -> "audio/x-m4a"
// else -> outMimeType
// }
tempFileOpener(
tempFile,
outMimeType,
isImage = false,
)
}
}
error("audio conversion is not yet supported.")
}
}

7
app/src/main/java/jp/juggler/subwaytooter/util/AttachmentUploader.kt
View File

@ -6,6 +6,7 @@ import android.os.Handler
import android.os.SystemClock
import androidx.annotation.WorkerThread
import androidx.appcompat.app.AppCompatActivity
import androidx.media3.common.MimeTypes
import jp.juggler.subwaytooter.R
import jp.juggler.subwaytooter.api.TootApiCallback
import jp.juggler.subwaytooter.api.TootApiClient
@ -295,15 +296,15 @@ class AttachmentUploader(
val isAccepteble = instance.configuration
?.jsonObject("media_attachments")
?.jsonArray("supported_mime_types")
?.contains(mimeType)
?.contains(opener.mimeType)
?: when (instance.instanceType) {
InstanceType.Pixelfed -> acceptableMimeTypesPixelfed
else -> acceptableMimeTypes
}.contains(mimeType)
}.contains(opener.mimeType)
if (!isAccepteble) {
return TootApiResult(
safeContext.getString(R.string.mime_type_not_acceptable, mimeType)
safeContext.getString(R.string.mime_type_not_acceptable, opener.mimeType)
)
}

4
base/build.gradle
View File

@ -98,7 +98,9 @@ dependencies {
api "org.jetbrains.kotlinx:kotlinx-datetime:0.4.0"
api "org.jetbrains.kotlinx:kotlinx-serialization-json:1.4.1"
api "ru.gildor.coroutines:kotlin-coroutines-okhttp:1.0"
api "androidx.media3:media3-transformer:1.0.1"
api "androidx.media3:media3-effect:1.0.1"
api "androidx.media3:media3-common:1.0.1"
// commons-codecをapiにすると、端末上の古いjarが使われてしまう
// declaration of 'org.apache.commons.codec.binary.Base64' appears in /system/framework/org.apache.http.legacy.jar)
androidTestImplementation "commons-codec:commons-codec:1.15"

112
base/src/main/java/jp/juggler/media/AudioTranscoder.kt Normal file
View File

@ -0,0 +1,112 @@
package jp.juggler.media
import android.content.Context
import android.net.Uri
import android.os.Looper
import androidx.media3.common.MediaItem
import androidx.media3.common.MimeTypes
import androidx.media3.transformer.TransformationException
import androidx.media3.transformer.TransformationRequest
import androidx.media3.transformer.TransformationResult
import androidx.media3.transformer.Transformer
import jp.juggler.util.log.LogCategory
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.ExperimentalCoroutinesApi
import kotlinx.coroutines.suspendCancellableCoroutine
import kotlinx.coroutines.withContext
import java.io.File
import java.io.FileOutputStream
import java.nio.file.Files
import java.nio.file.Paths
import java.util.UUID
import kotlin.coroutines.resumeWithException
private val log = LogCategory("transcodeAudio")
val generateTempFileLock = Any()
fun Context.generateTempFile(prefix: String) =
synchronized(generateTempFileLock) {
val cacheDir = externalCacheDir ?: cacheDir ?: error("missing cacheDir")
cacheDir.mkdirs()
val path = Paths.get(cacheDir.canonicalPath)
if (!Files.isDirectory(path)) error("cacheDir is not directory. $cacheDir")
if (!Files.isWritable(path)) error("cacheDir is not writable. $cacheDir")
// 重複しない一時ファイル名を探す
var tempFile: File
do {
tempFile = File(cacheDir, "$prefix-${UUID.randomUUID()}")
} while (tempFile.exists())
// すぐにファイルを作成する
FileOutputStream(tempFile).use {}
tempFile
}
@OptIn(ExperimentalCoroutinesApi::class)
@androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class)
suspend fun transcodeAudio(
context: Context,
inUri: Uri,
inMimeType: String,
): Pair<File, String> {
val inputMediaItem = MediaItem.fromUri(inUri)
// トランスコードに指定するmimeType
val encodeMimeType = MimeTypes.AUDIO_AAC
// MediaStore登録に使うmimeType。 audio/mp4a-latm だと失敗するのホント困る
val storeMimeType = "audio/mp4"
val tmpFile = context.generateTempFile("transcodeAudio")
// Transformerは単一スレッドで処理する要件
val result: TransformationResult = withContext(Dispatchers.Main.immediate) {
val looper = Looper.getMainLooper()
suspendCancellableCoroutine { cont ->
val transformerListener = object : Transformer.Listener {
override fun onTransformationCompleted(
inputMediaItem: MediaItem,
transformationResult: TransformationResult,
) {
log.i("onTransformationCompleted inputMediaItem=$inputMediaItem transformationResult=$transformationResult")
if (cont.isActive) cont.resume(transformationResult) {}
}
override fun onTransformationError(
inputMediaItem: MediaItem,
exception: TransformationException,
) {
log.e(
exception,
"onTransformationError inputMediaItem=$inputMediaItem"
)
if (cont.isActive) cont.resumeWithException(exception)
}
override fun onFallbackApplied(
inputMediaItem: MediaItem,
originalTransformationRequest: TransformationRequest,
fallbackTransformationRequest: TransformationRequest,
) {
log.i("onFallbackApplied inputMediaItem=$inputMediaItem original=$originalTransformationRequest fallback=$fallbackTransformationRequest")
}
}
val transformer = Transformer.Builder(context)
.setLooper(looper)
.setTransformationRequest(
TransformationRequest.Builder()
.setAudioMimeType(encodeMimeType)
.build()
)
.setRemoveVideo(true)
.setRemoveAudio(false)
.addListener(transformerListener)
.build()
transformer.startTransformation(inputMediaItem, tmpFile.canonicalPath)
cont.invokeOnCancellation {
transformer.cancel()
}
}
}
result.run {
log.i("result: durationMs=$durationMs, fileSizeBytes=$fileSizeBytes, averageAudioBitrate=$averageAudioBitrate, averageVideoBitrate=$averageVideoBitrate, videoFrameCount=$videoFrameCount")
}
return Pair(tmpFile, storeMimeType)
}