filippodb/SchildiChat-android
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Active call (with PIP) , in Room and Home

Browse Source
This commit is contained in:
Valere 2020-06-19 18:54:39 +02:00
parent 60998c9146
commit 17cf3fd7ad
7 changed files with 298 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

98
vector/src/main/java/im/vector/riotx/core/ui/views/ActiveCallViewHolder.kt Normal file
View File

@ -0,0 +1,98 @@
/*
* Copyright (c) 2020 New Vector Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package im.vector.riotx.core.ui.views
import android.view.View
import androidx.cardview.widget.CardView
import androidx.core.view.isVisible
import im.vector.matrix.android.api.session.call.CallState
import im.vector.matrix.android.api.session.call.EglUtils
import im.vector.matrix.android.api.session.call.MxCall
import im.vector.riotx.core.utils.DebouncedClickListener
import im.vector.riotx.features.call.WebRtcPeerConnectionManager
import org.webrtc.RendererCommon
import org.webrtc.SurfaceViewRenderer
class ActiveCallViewHolder() {
private var activeCallPiP: SurfaceViewRenderer? = null
private var activeCallView: ActiveCallView? = null
private var pipWrapper: CardView? = null
private var activeCallPipInitialized = false
fun updateCall(activeCall: MxCall?, webRtcPeerConnectionManager: WebRtcPeerConnectionManager) {
val hasActiveCall = activeCall?.state is CallState.Connected
if (hasActiveCall) {
val isVideoCall = activeCall?.isVideoCall == true
if (isVideoCall) initIfNeeded()
activeCallView?.isVisible = !isVideoCall
pipWrapper?.isVisible = isVideoCall
activeCallPiP?.isVisible = isVideoCall
activeCallPiP?.let {
webRtcPeerConnectionManager.attachViewRenderers(null, it, null)
}
} else {
activeCallView?.isVisible = false
activeCallPiP?.isVisible = false
pipWrapper?.isVisible = false
activeCallPiP?.let {
webRtcPeerConnectionManager.detachRenderers(listOf(it))
}
}
}
private fun initIfNeeded() {
if (!activeCallPipInitialized && activeCallPiP != null) {
activeCallPiP?.setScalingType(RendererCommon.ScalingType.SCALE_ASPECT_FILL)
EglUtils.rootEglBase?.let { eglBase ->
activeCallPiP?.init(eglBase.eglBaseContext, null)
activeCallPiP?.setScalingType(RendererCommon.ScalingType.SCALE_ASPECT_BALANCED)
activeCallPiP?.setEnableHardwareScaler(true /* enabled */)
activeCallPiP?.setZOrderMediaOverlay(true)
activeCallPipInitialized
}
}
}
fun bind(activeCallPiP: SurfaceViewRenderer, activeCallView: ActiveCallView, pipWrapper: CardView, interactionListener: ActiveCallView.Callback) {
this.activeCallPiP = activeCallPiP
this.activeCallView = activeCallView
this.pipWrapper = pipWrapper
this.activeCallView?.callback = interactionListener
pipWrapper.setOnClickListener(
DebouncedClickListener(View.OnClickListener { _ ->
interactionListener.onTapToReturnToCall()
})
)
}
fun unBind(webRtcPeerConnectionManager: WebRtcPeerConnectionManager) {
activeCallPiP?.let {
webRtcPeerConnectionManager.detachRenderers(listOf(it))
}
if (activeCallPipInitialized) {
activeCallPiP?.release()
}
this.activeCallView?.callback = null
pipWrapper?.setOnClickListener(null)
activeCallPiP = null
activeCallView = null
pipWrapper = null
}
}

2
vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt
View File

@ -210,7 +210,7 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis
}
override fun onDestroy() {
peerConnectionManager.detachRenderers()
peerConnectionManager.detachRenderers(listOf(pipRenderer, fullscreenRenderer))
if (surfaceRenderersAreInitialized) {
pipRenderer.release()
fullscreenRenderer.release()

124
vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt
View File

@ -175,8 +175,28 @@ class WebRtcPeerConnectionManager @Inject constructor(
}
}
var localSurfaceRenderer: WeakReference<SurfaceViewRenderer>? = null
var remoteSurfaceRenderer: WeakReference<SurfaceViewRenderer>? = null
var localSurfaceRenderer: MutableList<WeakReference<SurfaceViewRenderer>> = ArrayList()
var remoteSurfaceRenderer: MutableList<WeakReference<SurfaceViewRenderer>> = ArrayList()
fun addIfNeeded(renderer: SurfaceViewRenderer?, list: MutableList<WeakReference<SurfaceViewRenderer>>) {
if (renderer == null) return
val exists = list.firstOrNull() {
it.get() == renderer
} != null
if (!exists) {
list.add(WeakReference(renderer))
}
}
fun removeIfNeeded(renderer: SurfaceViewRenderer?, list: MutableList<WeakReference<SurfaceViewRenderer>>) {
if (renderer == null) return
val exists = list.indexOfFirst {
it.get() == renderer
}
if (exists != -1) {
list.add(WeakReference(renderer))
}
}
var currentCall: CallContext? = null
set(value) {
@ -279,10 +299,12 @@ class WebRtcPeerConnectionManager @Inject constructor(
})
}
fun attachViewRenderers(localViewRenderer: SurfaceViewRenderer, remoteViewRenderer: SurfaceViewRenderer, mode: String?) {
fun attachViewRenderers(localViewRenderer: SurfaceViewRenderer?, remoteViewRenderer: SurfaceViewRenderer, mode: String?) {
Timber.v("## VOIP attachViewRenderers localRendeder $localViewRenderer / $remoteViewRenderer")
this.localSurfaceRenderer = WeakReference(localViewRenderer)
this.remoteSurfaceRenderer = WeakReference(remoteViewRenderer)
// this.localSurfaceRenderer = WeakReference(localViewRenderer)
// this.remoteSurfaceRenderer = WeakReference(remoteViewRenderer)
addIfNeeded(localViewRenderer, this.localSurfaceRenderer)
addIfNeeded(remoteViewRenderer, this.remoteSurfaceRenderer)
// The call is going to resume from background, we can reduce notif
currentCall?.mxCall
@ -482,15 +504,21 @@ class WebRtcPeerConnectionManager @Inject constructor(
private fun attachViewRenderersInternal() {
// render local video in pip view
localSurfaceRenderer?.get()?.let { pipSurface ->
pipSurface.setMirror(true)
currentCall?.localVideoTrack?.addSink(pipSurface)
localSurfaceRenderer.forEach {
it.get()?.let { pipSurface ->
pipSurface.setMirror(true)
// no need to check if already added, addSink is checking that
currentCall?.localVideoTrack?.addSink(pipSurface)
}
}
// If remote track exists, then sink it to surface
remoteSurfaceRenderer?.get()?.let { participantSurface ->
currentCall?.remoteVideoTrack?.let {
it.addSink(participantSurface)
remoteSurfaceRenderer.forEach {
it.get()?.let { participantSurface ->
currentCall?.remoteVideoTrack?.let {
// no need to check if already added, addSink is checking that
it.addSink(participantSurface)
}
}
}
}
@ -505,35 +533,48 @@ class WebRtcPeerConnectionManager @Inject constructor(
}
}
fun detachRenderers() {
// The call is going to continue in background, so ensure notification is visible
currentCall?.mxCall
?.takeIf { it.state is CallState.Connected }
?.let { mxCall ->
// Start background service with notification
val name = sessionHolder.getSafeActiveSession()?.getUser(mxCall.otherUserId)?.getBestName()
?: mxCall.otherUserId
CallService.onOnGoingCallBackground(
context = context,
isVideo = mxCall.isVideoCall,
roomName = name,
roomId = mxCall.roomId,
matrixId = sessionHolder.getSafeActiveSession()?.myUserId ?: "",
callId = mxCall.callId
)
}
fun detachRenderers(renderes: List<SurfaceViewRenderer>?) {
Timber.v("## VOIP detachRenderers")
// currentCall?.localMediaStream?.let { currentCall?.peerConnection?.removeStream(it) }
localSurfaceRenderer?.get()?.let {
currentCall?.localVideoTrack?.removeSink(it)
if (renderes.isNullOrEmpty()) {
// remove all sinks
localSurfaceRenderer.forEach {
if (it.get() != null) currentCall?.localVideoTrack?.removeSink(it.get())
}
remoteSurfaceRenderer.forEach {
if (it.get() != null) currentCall?.remoteVideoTrack?.removeSink(it.get())
}
localSurfaceRenderer.clear()
remoteSurfaceRenderer.clear()
} else {
renderes.forEach {
removeIfNeeded(it, localSurfaceRenderer)
removeIfNeeded(it, remoteSurfaceRenderer)
// no need to check if it's in the track, removeSink is doing it
currentCall?.localVideoTrack?.removeSink(it)
currentCall?.remoteVideoTrack?.removeSink(it)
}
}
remoteSurfaceRenderer?.get()?.let {
currentCall?.remoteVideoTrack?.removeSink(it)
if (remoteSurfaceRenderer.isEmpty()) {
// The call is going to continue in background, so ensure notification is visible
currentCall?.mxCall
?.takeIf { it.state is CallState.Connected }
?.let { mxCall ->
// Start background service with notification
val name = sessionHolder.getSafeActiveSession()?.getUser(mxCall.otherUserId)?.getBestName()
?: mxCall.otherUserId
CallService.onOnGoingCallBackground(
context = context,
isVideo = mxCall.isVideoCall,
roomName = name,
roomId = mxCall.roomId,
matrixId = sessionHolder.getSafeActiveSession()?.myUserId ?: "",
callId = mxCall.callId
)
}
}
localSurfaceRenderer = null
remoteSurfaceRenderer = null
}
fun close() {
@ -946,7 +987,7 @@ class WebRtcPeerConnectionManager @Inject constructor(
remoteVideoTrack.setEnabled(true)
callContext.remoteVideoTrack = remoteVideoTrack
// sink to renderer if attached
remoteSurfaceRenderer?.get()?.let { remoteVideoTrack.addSink(it) }
remoteSurfaceRenderer.forEach { it.get()?.let { remoteVideoTrack.addSink(it) } }
}
}
}
@ -954,9 +995,12 @@ class WebRtcPeerConnectionManager @Inject constructor(
override fun onRemoveStream(stream: MediaStream) {
Timber.v("## VOIP StreamObserver onRemoveStream")
executor.execute {
remoteSurfaceRenderer?.get()?.let {
callContext.remoteVideoTrack?.removeSink(it)
}
// remoteSurfaceRenderer?.get()?.let {
// callContext.remoteVideoTrack?.removeSink(it)
// }
remoteSurfaceRenderer
.mapNotNull { it.get() }
.forEach { callContext.remoteVideoTrack?.removeSink(it) }
callContext.remoteVideoTrack = null
}
}

52
vector/src/main/java/im/vector/riotx/features/home/HomeDetailFragment.kt
View File

@ -37,7 +37,12 @@ import im.vector.riotx.core.glide.GlideApp
import im.vector.riotx.core.platform.ToolbarConfigurable
import im.vector.riotx.core.platform.VectorBaseActivity
import im.vector.riotx.core.platform.VectorBaseFragment
import im.vector.riotx.core.ui.views.ActiveCallView
import im.vector.riotx.core.ui.views.ActiveCallViewHolder
import im.vector.riotx.core.ui.views.KeysBackupBanner
import im.vector.riotx.features.call.SharedActiveCallViewModel
import im.vector.riotx.features.call.VectorCallActivity
import im.vector.riotx.features.call.WebRtcPeerConnectionManager
import im.vector.riotx.features.home.room.list.RoomListFragment
import im.vector.riotx.features.home.room.list.RoomListParams
import im.vector.riotx.features.home.room.list.UnreadCounterBadgeView
@ -46,6 +51,11 @@ import im.vector.riotx.features.popup.VerificationVectorAlert
import im.vector.riotx.features.settings.VectorSettingsActivity.Companion.EXTRA_DIRECT_ACCESS_SECURITY_PRIVACY_MANAGE_SESSIONS
import im.vector.riotx.features.workers.signout.SignOutViewModel
import kotlinx.android.synthetic.main.fragment_home_detail.*
import kotlinx.android.synthetic.main.fragment_home_detail.activeCallPiP
import kotlinx.android.synthetic.main.fragment_home_detail.activeCallPiPWrap
import kotlinx.android.synthetic.main.fragment_home_detail.activeCallView
import kotlinx.android.synthetic.main.fragment_home_detail.syncStateView
import kotlinx.android.synthetic.main.fragment_room_detail.*
import timber.log.Timber
import javax.inject.Inject
@ -56,8 +66,9 @@ private const val INDEX_ROOMS = 2
class HomeDetailFragment @Inject constructor(
val homeDetailViewModelFactory: HomeDetailViewModel.Factory,
private val avatarRenderer: AvatarRenderer,
private val alertManager: PopupAlertManager
) : VectorBaseFragment(), KeysBackupBanner.Delegate {
private val alertManager: PopupAlertManager,
private val webRtcPeerConnectionManager: WebRtcPeerConnectionManager
) : VectorBaseFragment(), KeysBackupBanner.Delegate, ActiveCallView.Callback {
private val unreadCounterBadgeViews = arrayListOf<UnreadCounterBadgeView>()
@ -65,16 +76,21 @@ class HomeDetailFragment @Inject constructor(
private val unknownDeviceDetectorSharedViewModel: UnknownDeviceDetectorSharedViewModel by activityViewModel()
private lateinit var sharedActionViewModel: HomeSharedActionViewModel
private lateinit var sharedCallActionViewModel: SharedActiveCallViewModel
override fun getLayoutResId() = R.layout.fragment_home_detail
private val activeCallViewHolder = ActiveCallViewHolder()
override fun onViewCreated(view: View, savedInstanceState: Bundle?) {
super.onViewCreated(view, savedInstanceState)
sharedActionViewModel = activityViewModelProvider.get(HomeSharedActionViewModel::class.java)
sharedCallActionViewModel = activityViewModelProvider.get(SharedActiveCallViewModel::class.java)
setupBottomNavigationView()
setupToolbar()
setupKeysBackupBanner()
setupActiveCallView()
withState(viewModel) {
// Update the navigation view if needed (for when we restore the tabs)
@ -105,6 +121,13 @@ class HomeDetailFragment @Inject constructor(
}
}
}
sharedCallActionViewModel
.activeCall
.observe(viewLifecycleOwner, Observer {
activeCallViewHolder.updateCall(it, webRtcPeerConnectionManager)
invalidateOptionsMenu()
})
}
private fun promptForNewUnknownDevices(uid: String, state: UnknownDevicesState, newest: DeviceInfo) {
@ -203,6 +226,15 @@ class HomeDetailFragment @Inject constructor(
homeKeysBackupBanner.delegate = this
}
private fun setupActiveCallView() {
activeCallViewHolder.bind(
activeCallPiP,
activeCallView,
activeCallPiPWrap,
this
)
}
private fun setupToolbar() {
val parentActivity = vectorBaseActivity
if (parentActivity is ToolbarConfigurable) {
@ -283,4 +315,20 @@ class HomeDetailFragment @Inject constructor(
RoomListDisplayMode.ROOMS -> R.id.bottom_action_rooms
else -> R.id.bottom_action_home
}
override fun onTapToReturnToCall() {
sharedCallActionViewModel.activeCall.value?.let { call ->
VectorCallActivity.newIntent(
context = requireContext(),
callId = call.callId,
roomId = call.roomId,
otherUserId = call.otherUserId,
isIncomingCall = !call.isOutgoing,
isVideoCall = call.isVideoCall,
mode = null
).let {
startActivity(it)
}
}
}
}

18
vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt
View File

@ -63,7 +63,6 @@ import com.jakewharton.rxbinding3.widget.textChanges
import im.vector.matrix.android.api.MatrixCallback
import im.vector.matrix.android.api.permalinks.PermalinkFactory
import im.vector.matrix.android.api.session.Session
import im.vector.matrix.android.api.session.call.CallState
import im.vector.matrix.android.api.session.content.ContentAttachmentData
import im.vector.matrix.android.api.session.events.model.Event
import im.vector.matrix.android.api.session.events.model.toModel
@ -103,6 +102,7 @@ import im.vector.riotx.core.intent.getMimeTypeFromUri
import im.vector.riotx.core.platform.VectorBaseFragment
import im.vector.riotx.core.resources.ColorProvider
import im.vector.riotx.core.ui.views.ActiveCallView
import im.vector.riotx.core.ui.views.ActiveCallViewHolder
import im.vector.riotx.core.ui.views.JumpToReadMarkerView
import im.vector.riotx.core.ui.views.NotificationAreaView
import im.vector.riotx.core.utils.Debouncer
@ -136,6 +136,7 @@ import im.vector.riotx.features.attachments.preview.AttachmentsPreviewArgs
import im.vector.riotx.features.attachments.toGroupedContentAttachmentData
import im.vector.riotx.features.call.SharedActiveCallViewModel
import im.vector.riotx.features.call.VectorCallActivity
import im.vector.riotx.features.call.WebRtcPeerConnectionManager
import im.vector.riotx.features.command.Command
import im.vector.riotx.features.crypto.keysbackup.restore.KeysBackupRestoreActivity
import im.vector.riotx.features.crypto.util.toImageRes
@ -205,7 +206,8 @@ class RoomDetailFragment @Inject constructor(
val roomDetailViewModelFactory: RoomDetailViewModel.Factory,
private val eventHtmlRenderer: EventHtmlRenderer,
private val vectorPreferences: VectorPreferences,
private val colorProvider: ColorProvider) :
private val colorProvider: ColorProvider,
private val webRtcPeerConnectionManager: WebRtcPeerConnectionManager) :
VectorBaseFragment(),
TimelineEventController.Callback,
VectorInviteView.Callback,
@ -270,6 +272,7 @@ class RoomDetailFragment @Inject constructor(
private lateinit var attachmentTypeSelector: AttachmentTypeSelectorView
private var lockSendButton = false
private val activeCallViewHolder = ActiveCallViewHolder()
override fun onViewCreated(view: View, savedInstanceState: Bundle?) {
super.onViewCreated(view, savedInstanceState)
@ -301,8 +304,7 @@ class RoomDetailFragment @Inject constructor(
sharedCallActionViewModel
.activeCall
.observe(viewLifecycleOwner, Observer {
val hasActiveCall = it?.state is CallState.Connected
activeCallView.isVisible = hasActiveCall
activeCallViewHolder.updateCall(it, webRtcPeerConnectionManager)
invalidateOptionsMenu()
})
@ -407,6 +409,7 @@ class RoomDetailFragment @Inject constructor(
}
override fun onDestroy() {
activeCallViewHolder.unBind(webRtcPeerConnectionManager)
roomDetailViewModel.handle(RoomDetailAction.ExitTrackingUnreadMessagesState)
super.onDestroy()
}
@ -437,7 +440,12 @@ class RoomDetailFragment @Inject constructor(
}
private fun setupActiveCallView() {
activeCallView.callback = this
activeCallViewHolder.bind(
activeCallPiP,
activeCallView,
activeCallPiPWrap,
this
)
}
private fun navigateToEvent(action: RoomDetailViewEvents.NavigateToEvent) {

32
vector/src/main/res/layout/fragment_home_detail.xml
View File

@ -63,13 +63,43 @@
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintTop_toBottomOf="@id/syncStateView" />
<im.vector.riotx.core.ui.views.ActiveCallView
android:id="@+id/activeCallView"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:visibility="gone"
app:layout_constraintTop_toBottomOf="@id/homeKeysBackupBanner"
tools:visibility="visible" />
<androidx.fragment.app.FragmentContainerView
android:id="@+id/roomListContainer"
android:layout_width="match_parent"
android:layout_height="0dp"
android:background="?riotx_header_panel_background"
app:layout_constraintBottom_toTopOf="@+id/bottomNavigationView"
app:layout_constraintTop_toBottomOf="@+id/homeKeysBackupBanner" />
app:layout_constraintTop_toBottomOf="@+id/activeCallView" />
<androidx.cardview.widget.CardView
android:id="@+id/activeCallPiPWrap"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginTop="8dp"
android:layout_marginEnd="8dp"
android:clickable="true"
android:focusable="true"
app:cardCornerRadius="16dp"
app:layout_constraintEnd_toEndOf="parent"
app:layout_constraintTop_toBottomOf="@id/activeCallView">
<org.webrtc.SurfaceViewRenderer
android:id="@+id/activeCallPiP"
android:layout_width="120dp"
android:layout_height="120dp"
android:visibility="gone"
tools:visibility="visible" />
</androidx.cardview.widget.CardView>
<com.google.android.material.bottomnavigation.BottomNavigationView
android:id="@+id/bottomNavigationView"

21
vector/src/main/res/layout/fragment_room_detail.xml
View File

@ -182,6 +182,27 @@
app:barrierDirection="top"
app:constraint_referenced_ids="composerLayout,notificationAreaView" />
<androidx.cardview.widget.CardView
android:id="@+id/activeCallPiPWrap"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginTop="8dp"
android:layout_marginEnd="8dp"
android:clickable="true"
android:focusable="true"
app:cardCornerRadius="16dp"
app:layout_constraintEnd_toEndOf="parent"
app:layout_constraintTop_toBottomOf="@id/activeCallView">
<org.webrtc.SurfaceViewRenderer
android:id="@+id/activeCallPiP"
android:layout_width="120dp"
android:layout_height="120dp"
android:visibility="gone"
tools:visibility="visible" />
</androidx.cardview.widget.CardView>
<im.vector.riotx.core.platform.BadgeFloatingActionButton
android:id="@+id/jumpToBottomView"
android:layout_width="wrap_content"