From e9c81ca98fea45ced23afa32185d3a29762ded4a Mon Sep 17 00:00:00 2001 From: Florian Renaud Date: Wed, 19 Oct 2022 18:07:44 +0200 Subject: [PATCH] VoiceBroadcastPlayer - Live listening --- .../sdk/api/session/events/model/Event.kt | 2 +- .../room/timeline/TimelineEventDataSource.kt | 10 +- .../voicebroadcast/VoiceBroadcastHelper.kt | 2 +- .../voicebroadcast/VoiceBroadcastPlayer.kt | 236 +++++++++++++++--- .../usecase/GetVoiceBroadcastStateUseCase.kt | 41 +++ 5 files changed, 249 insertions(+), 42 deletions(-) create mode 100644 vector/src/main/java/im/vector/app/features/voicebroadcast/usecase/GetVoiceBroadcastStateUseCase.kt diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/events/model/Event.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/events/model/Event.kt index 71daf4cc4f..1f16041b54 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/events/model/Event.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/events/model/Event.kt @@ -401,7 +401,7 @@ fun Event.getRelationContent(): RelationDefaultContent? { when (getClearType()) { EventType.STICKER -> getClearContent().toModel()?.relatesTo in EventType.BEACON_LOCATION_DATA -> getClearContent().toModel()?.relatesTo - else -> null + else -> getClearContent()?.get("m.relates_to")?.toContent().toModel() } } } diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/timeline/TimelineEventDataSource.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/timeline/TimelineEventDataSource.kt index 20094e4be8..2d6082f9b5 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/timeline/TimelineEventDataSource.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/timeline/TimelineEventDataSource.kt @@ -22,6 +22,8 @@ import io.realm.Sort import org.matrix.android.sdk.api.session.events.model.getRelationContent import org.matrix.android.sdk.api.session.events.model.isImageMessage import org.matrix.android.sdk.api.session.events.model.isVideoMessage +import org.matrix.android.sdk.api.session.events.model.toModel +import org.matrix.android.sdk.api.session.room.model.message.MessageContent import org.matrix.android.sdk.api.session.room.timeline.TimelineEvent import org.matrix.android.sdk.api.util.Optional import org.matrix.android.sdk.internal.database.RealmSessionProvider @@ -74,7 +76,13 @@ internal class TimelineEventDataSource @Inject constructor( .distinct(TimelineEventEntityFields.EVENT_ID) .findAll() .mapNotNull { - timelineEventMapper.map(it).takeIf { it.root.getRelationContent()?.takeIf { it.type == eventType && it.eventId == eventId } != null } + timelineEventMapper.map(it) + .takeIf { + val isEventRelatedTo = it.root.getRelationContent()?.takeIf { it.type == eventType && it.eventId == eventId } != null + val isContentRelatedTo = it.root.getClearContent()?.toModel() + ?.relatesTo?.takeIf { it.type == eventType && it.eventId == eventId } != null + isEventRelatedTo || isContentRelatedTo + } } } } diff --git a/vector/src/main/java/im/vector/app/features/voicebroadcast/VoiceBroadcastHelper.kt b/vector/src/main/java/im/vector/app/features/voicebroadcast/VoiceBroadcastHelper.kt index b967afa9cb..58e7de7f32 100644 --- a/vector/src/main/java/im/vector/app/features/voicebroadcast/VoiceBroadcastHelper.kt +++ b/vector/src/main/java/im/vector/app/features/voicebroadcast/VoiceBroadcastHelper.kt @@ -40,7 +40,7 @@ class VoiceBroadcastHelper @Inject constructor( suspend fun stopVoiceBroadcast(roomId: String) = stopVoiceBroadcastUseCase.execute(roomId) - fun playOrResumePlayback(roomId: String, eventId: String) = voiceBroadcastPlayer.play(roomId, eventId) + fun playOrResumePlayback(roomId: String, eventId: String) = voiceBroadcastPlayer.playOrResume(roomId, eventId) fun pausePlayback() = voiceBroadcastPlayer.pause() diff --git a/vector/src/main/java/im/vector/app/features/voicebroadcast/VoiceBroadcastPlayer.kt b/vector/src/main/java/im/vector/app/features/voicebroadcast/VoiceBroadcastPlayer.kt index 72ec181966..7f5e13504e 100644 --- a/vector/src/main/java/im/vector/app/features/voicebroadcast/VoiceBroadcastPlayer.kt +++ b/vector/src/main/java/im/vector/app/features/voicebroadcast/VoiceBroadcastPlayer.kt @@ -20,13 +20,19 @@ import android.media.AudioAttributes import android.media.MediaPlayer import im.vector.app.core.di.ActiveSessionHolder import im.vector.app.features.home.room.detail.timeline.helper.AudioMessagePlaybackTracker -import im.vector.app.features.home.room.detail.timeline.helper.AudioMessagePlaybackTracker.Listener.State import im.vector.app.features.voice.VoiceFailure +import im.vector.app.features.voicebroadcast.model.VoiceBroadcastState +import im.vector.app.features.voicebroadcast.model.asVoiceBroadcastEvent +import im.vector.app.features.voicebroadcast.usecase.GetVoiceBroadcastStateUseCase import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.flow.launchIn +import kotlinx.coroutines.flow.mapNotNull +import kotlinx.coroutines.flow.onEach import kotlinx.coroutines.launch -import org.matrix.android.sdk.api.extensions.orFalse +import org.matrix.android.sdk.api.query.QueryStringValue import org.matrix.android.sdk.api.session.events.model.RelationType import org.matrix.android.sdk.api.session.events.model.getRelationContent import org.matrix.android.sdk.api.session.getRoom @@ -34,6 +40,11 @@ import org.matrix.android.sdk.api.session.room.Room import org.matrix.android.sdk.api.session.room.model.message.MessageAudioContent import org.matrix.android.sdk.api.session.room.model.message.MessageAudioEvent import org.matrix.android.sdk.api.session.room.model.message.asMessageAudioEvent +import org.matrix.android.sdk.api.session.room.timeline.Timeline +import org.matrix.android.sdk.api.session.room.timeline.TimelineEvent +import org.matrix.android.sdk.api.session.room.timeline.TimelineSettings +import org.matrix.android.sdk.flow.flow +import org.matrix.android.sdk.flow.unwrap import timber.log.Timber import javax.inject.Inject import javax.inject.Singleton @@ -42,62 +53,117 @@ import javax.inject.Singleton class VoiceBroadcastPlayer @Inject constructor( private val sessionHolder: ActiveSessionHolder, private val playbackTracker: AudioMessagePlaybackTracker, + private val getVoiceBroadcastStateUseCase: GetVoiceBroadcastStateUseCase, ) { - private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default) private val session get() = sessionHolder.getActiveSession() + private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default) + private var voiceBroadcastStateJob: Job? = null + private var currentTimeline: Timeline? = null + set(value) { + field?.removeAllListeners() + field?.dispose() + field = value + } + + private val mediaPlayerListener = MediaPlayerListener() + private var timelineListener: TimelineListener? = null + private var currentMediaPlayer: MediaPlayer? = null - private var currentPlayingIndex: Int = -1 + private var nextMediaPlayer: MediaPlayer? = null + set(value) { + field = value + currentMediaPlayer?.setNextMediaPlayer(value) + } + private var currentSequence: Int? = null private var playlist = emptyList() private val currentVoiceBroadcastId - get() = playlist.getOrNull(currentPlayingIndex)?.root?.getRelationContent()?.eventId + get() = playlist.firstOrNull()?.root?.getRelationContent()?.eventId - private val mediaPlayerListener = MediaPlayerListener() - - fun play(roomId: String, eventId: String) { - val room = session.getRoom(roomId) ?: error("Unknown roomId: $roomId") + private var state: State = State.IDLE + set(value) { + Timber.w("## VoiceBroadcastPlayer state: $field -> $value") + field = value + } + fun playOrResume(roomId: String, eventId: String) { + val hasChanged = currentVoiceBroadcastId != eventId when { - currentVoiceBroadcastId != eventId -> { - stop() - updatePlaylist(room, eventId) - startPlayback() - } - playbackTracker.getPlaybackState(eventId) is State.Playing -> pause() - else -> resumePlayback() + hasChanged -> startPlayback(roomId, eventId) + state == State.PAUSED -> resumePlayback() + else -> Unit } } fun pause() { currentMediaPlayer?.pause() currentVoiceBroadcastId?.let { playbackTracker.pausePlayback(it) } + state = State.PAUSED } fun stop() { + // Stop playback currentMediaPlayer?.stop() currentVoiceBroadcastId?.let { playbackTracker.stopPlayback(it) } + + // Release current player release(currentMediaPlayer) + currentMediaPlayer = null + + // Release next player + release(nextMediaPlayer) + nextMediaPlayer = null + + // Do not observe anymore voice broadcast state changes + voiceBroadcastStateJob?.cancel() + voiceBroadcastStateJob = null + + // In case of live broadcast, stop observing new chunks + currentTimeline?.dispose() + currentTimeline?.removeAllListeners() + currentTimeline = null + timelineListener = null + + // Update state + state = State.IDLE + + // Clear playlist playlist = emptyList() - currentPlayingIndex = -1 + currentSequence = null } - private fun updatePlaylist(room: Room, eventId: String) { - val timelineEvents = room.timelineService().getTimelineEventsRelatedTo(RelationType.REFERENCE, eventId) - val audioEvents = timelineEvents.mapNotNull { it.root.asMessageAudioEvent() } - playlist = audioEvents.sortedBy { it.getVoiceBroadcastChunk()?.sequence?.toLong() ?: it.root.originServerTs } + private fun startPlayback(roomId: String, eventId: String) { + val room = session.getRoom(roomId) ?: error("Unknown roomId: $roomId") + + // Stop listening previous voice broadcast if any + if (state != State.IDLE) stop() + + state = State.BUFFERING + + val voiceBroadcastState = getVoiceBroadcastStateUseCase.execute(roomId, eventId) + if (voiceBroadcastState == VoiceBroadcastState.STOPPED) { + // Get static playlist + updatePlaylist(getExistingChunks(room, eventId)) + startPlayback(false) + } else { + playLiveVoiceBroadcast(room, eventId) + } } - private fun startPlayback() { - val content = playlist.firstOrNull()?.content ?: run { Timber.w("## VoiceBroadcastPlayer: No content to play"); return } + private fun startPlayback(isLive: Boolean) { + val event = if (isLive) playlist.lastOrNull() else playlist.firstOrNull() + val content = event?.content ?: run { Timber.w("## VoiceBroadcastPlayer: No content to play"); return } + val sequence = event.getVoiceBroadcastChunk()?.sequence coroutineScope.launch { try { currentMediaPlayer = prepareMediaPlayer(content) currentMediaPlayer?.start() - currentPlayingIndex = 0 currentVoiceBroadcastId?.let { playbackTracker.startPlayback(it) } - prepareNextFile() + currentSequence = sequence + state = State.PLAYING + nextMediaPlayer = prepareNextMediaPlayer() } catch (failure: Throwable) { Timber.e(failure, "Unable to start playback") throw VoiceFailure.UnableToPlay(failure) @@ -105,19 +171,68 @@ class VoiceBroadcastPlayer @Inject constructor( } } + private fun playLiveVoiceBroadcast(room: Room, eventId: String) { + val voiceBroadcastEvent = room.timelineService().getTimelineEvent(eventId)?.root?.asVoiceBroadcastEvent() + ?: error("Cannot retrieve voice broadcast $eventId") + updatePlaylist(getExistingChunks(room, eventId)) + startPlayback(true) + room.flow() + .liveStateEvent(VoiceBroadcastConstants.STATE_ROOM_VOICE_BROADCAST_INFO, QueryStringValue.Equals(voiceBroadcastEvent.root.stateKey!!)) + .unwrap() + .mapNotNull { it.asVoiceBroadcastEvent()?.content?.voiceBroadcastState } + .onEach { state -> + when (state) { + VoiceBroadcastState.STARTED, + VoiceBroadcastState.PAUSED, + VoiceBroadcastState.RESUMED -> { + observeIncomingChunks(room, eventId) + } + VoiceBroadcastState.STOPPED -> { + currentTimeline?.dispose() + currentTimeline?.removeAllListeners() + currentTimeline = null + } + } + } + .launchIn(coroutineScope) + } + + private fun getExistingChunks(room: Room, eventId: String): List { + return room.timelineService().getTimelineEventsRelatedTo(RelationType.REFERENCE, eventId) + .mapNotNull { it.root.asMessageAudioEvent() } + .filter { it.isVoiceBroadcast() } + } + + private fun observeIncomingChunks(room: Room, eventId: String) { + // Fixme this is probably not necessary here + currentTimeline?.dispose() + currentTimeline?.removeAllListeners() + currentTimeline = room.timelineService().createTimeline(null, TimelineSettings(5)).also { timeline -> + timelineListener = TimelineListener(eventId).also { timeline.addListener(it) } + timeline.start() + } + } + private fun resumePlayback() { currentMediaPlayer?.start() currentVoiceBroadcastId?.let { playbackTracker.startPlayback(it) } + state = State.PLAYING } - private suspend fun prepareNextFile() { - val nextContent = playlist.getOrNull(currentPlayingIndex + 1)?.content - if (nextContent == null) { - currentMediaPlayer?.setOnCompletionListener(mediaPlayerListener) - } else { - val nextMediaPlayer = prepareMediaPlayer(nextContent) - currentMediaPlayer?.setNextMediaPlayer(nextMediaPlayer) - } + private fun updatePlaylist(playlist: List) { + this.playlist = playlist.sortedBy { it.getVoiceBroadcastChunk()?.sequence?.toLong() ?: it.root.originServerTs } + } + + private fun getNextAudioContent(): MessageAudioContent? { + val nextSequence = currentSequence?.plus(1) + ?: timelineListener?.let { playlist.lastOrNull()?.sequence } + ?: 1 + return playlist.find { it.getVoiceBroadcastChunk()?.sequence == nextSequence }?.content + } + + private suspend fun prepareNextMediaPlayer(): MediaPlayer? { + val nextContent = getNextAudioContent() ?: return null + return prepareMediaPlayer(nextContent) } private suspend fun prepareMediaPlayer(messageAudioContent: MessageAudioContent): MediaPlayer { @@ -141,6 +256,7 @@ class VoiceBroadcastPlayer @Inject constructor( setDataSource(fis.fd) setOnInfoListener(mediaPlayerListener) setOnErrorListener(mediaPlayerListener) + setOnCompletionListener(mediaPlayerListener) prepare() } } @@ -155,24 +271,59 @@ class VoiceBroadcastPlayer @Inject constructor( } } - inner class MediaPlayerListener : MediaPlayer.OnInfoListener, MediaPlayer.OnCompletionListener, MediaPlayer.OnErrorListener { + private inner class TimelineListener(private val voiceBroadcastId: String) : Timeline.Listener { + override fun onTimelineUpdated(snapshot: List) { + val currentSequences = playlist.map { it.sequence } + val newChunks = snapshot + .mapNotNull { timelineEvent -> + timelineEvent.root.asMessageAudioEvent() + ?.takeIf { it.isVoiceBroadcast() && it.getVoiceBroadcastEventId() == voiceBroadcastId && it.sequence !in currentSequences } + } + if (newChunks.isEmpty()) return + updatePlaylist(playlist + newChunks) + + when (state) { + State.PLAYING -> { + if (nextMediaPlayer == null) { + coroutineScope.launch { nextMediaPlayer = prepareNextMediaPlayer() } + } + } + State.PAUSED -> { + if (nextMediaPlayer == null) { + coroutineScope.launch { nextMediaPlayer = prepareNextMediaPlayer() } + } + } + State.BUFFERING -> { + val newMediaContent = getNextAudioContent() + if (newMediaContent != null) startPlayback(true) + } + State.IDLE -> startPlayback(true) + } + } + } + + private inner class MediaPlayerListener : MediaPlayer.OnInfoListener, MediaPlayer.OnCompletionListener, MediaPlayer.OnErrorListener { override fun onInfo(mp: MediaPlayer, what: Int, extra: Int): Boolean { when (what) { MediaPlayer.MEDIA_INFO_STARTED_AS_NEXT -> { release(currentMediaPlayer) currentMediaPlayer = mp - currentPlayingIndex++ - coroutineScope.launch { prepareNextFile() } + currentSequence = currentSequence?.plus(1) + coroutineScope.launch { nextMediaPlayer = prepareNextMediaPlayer() } } } return false } override fun onCompletion(mp: MediaPlayer) { - // Verify that a new media has not been set in the mean time - if (!currentMediaPlayer?.isPlaying.orFalse()) { - stop() + when { + timelineListener == null && nextMediaPlayer == null -> { + stop() + } + nextMediaPlayer == null -> { + state = State.BUFFERING + } } } @@ -181,4 +332,11 @@ class VoiceBroadcastPlayer @Inject constructor( return true } } + + enum class State { + PLAYING, + PAUSED, + BUFFERING, + IDLE + } } diff --git a/vector/src/main/java/im/vector/app/features/voicebroadcast/usecase/GetVoiceBroadcastStateUseCase.kt b/vector/src/main/java/im/vector/app/features/voicebroadcast/usecase/GetVoiceBroadcastStateUseCase.kt new file mode 100644 index 0000000000..5b3153ea40 --- /dev/null +++ b/vector/src/main/java/im/vector/app/features/voicebroadcast/usecase/GetVoiceBroadcastStateUseCase.kt @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2022 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.app.features.voicebroadcast.usecase + +import im.vector.app.features.voicebroadcast.model.VoiceBroadcastState +import im.vector.app.features.voicebroadcast.model.asVoiceBroadcastEvent +import org.matrix.android.sdk.api.session.Session +import org.matrix.android.sdk.api.session.events.model.RelationType +import org.matrix.android.sdk.api.session.getRoom +import timber.log.Timber +import javax.inject.Inject + +class GetVoiceBroadcastStateUseCase @Inject constructor( + private val session: Session, +) { + + fun execute(roomId: String, eventId: String): VoiceBroadcastState? { + val room = session.getRoom(roomId) ?: error("Unknown roomId: $roomId") + + Timber.d("## GetVoiceBroadcastStateUseCase: get voice broadcast state requested for $eventId") + + val initialEvent = room.timelineService().getTimelineEvent(eventId)?.root?.asVoiceBroadcastEvent() // Fallback to initial event + val relatedEvents = room.timelineService().getTimelineEventsRelatedTo(RelationType.REFERENCE, eventId).sortedBy { it.root.originServerTs } + val lastVoiceBroadcastEvent = relatedEvents.mapNotNull { it.root.asVoiceBroadcastEvent() }.lastOrNull() ?: initialEvent + return lastVoiceBroadcastEvent?.content?.voiceBroadcastState + } +}