filippodb
/
SchildiChat-android
mirror of https://github.com/SchildiChat/SchildiChat-android.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

VoIP: continue refactoring

This commit is contained in:
ganfra 2020-11-24 17:30:13 +01:00
parent d7f7aa09fc
commit 7620aa4264
20 changed files with 366 additions and 407 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/CallListener.kt
View File

@ -57,5 +57,8 @@ interface CallListener {
*/
fun onCallNegotiateReceived(callNegotiateContent: CallNegotiateContent)
/**
* Called when the call has been managed by an other session
*/
fun onCallManagedByOtherSession(callId: String)
}

3
matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/MxCall.kt
View File

@ -17,6 +17,7 @@
package org.matrix.android.sdk.api.session.call
import org.matrix.android.sdk.api.session.room.model.call.CallCandidate
import org.matrix.android.sdk.api.session.room.model.call.CallHangupContent
import org.matrix.android.sdk.api.util.Optional
interface MxCallDetail {
@ -66,7 +67,7 @@ interface MxCall : MxCallDetail {
/**
* End the call
*/
fun hangUp()
fun hangUp(reason: CallHangupContent.Reason? = null)
/**
* Start a call

4
matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/model/MxCallImpl.kt
View File

@ -134,12 +134,12 @@ internal class MxCallImpl(
state = CallState.Terminated
}
override fun hangUp() {
override fun hangUp(reason: CallHangupContent.Reason?) {
Timber.v("## VOIP hangup $callId")
CallHangupContent(
callId = callId,
partyId = ourPartyId,
reason = CallHangupContent.Reason.USER_HANGUP
reason = reason ?: CallHangupContent.Reason.USER_HANGUP
)
.let { createEventAndLocalEcho(type = EventType.CALL_HANGUP, roomId = roomId, content = it.toContent()) }
.also { eventSenderProcessor.postEvent(it) }

6
vector/src/main/java/im/vector/app/VectorApplication.kt
View File

@ -42,7 +42,7 @@ import im.vector.app.core.di.HasVectorInjector
import im.vector.app.core.di.VectorComponent
import im.vector.app.core.extensions.configureAndStart
import im.vector.app.core.rx.RxConfig
import im.vector.app.features.call.webrtc.WebRtcPeerConnectionManager
import im.vector.app.features.call.webrtc.WebRtcCallManager
import im.vector.app.features.configuration.VectorConfiguration
import im.vector.app.features.disclaimer.doNotShowDisclaimerDialog
import im.vector.app.features.lifecycle.VectorActivityLifecycleCallbacks
@ -90,7 +90,7 @@ class VectorApplication :
@Inject lateinit var rxConfig: RxConfig
@Inject lateinit var popupAlertManager: PopupAlertManager
@Inject lateinit var pinLocker: PinLocker
@Inject lateinit var webRtcPeerConnectionManager: WebRtcPeerConnectionManager
@Inject lateinit var callManager: WebRtcCallManager
lateinit var vectorComponent: VectorComponent
@ -175,7 +175,7 @@ class VectorApplication :
})
ProcessLifecycleOwner.get().lifecycle.addObserver(appStateHandler)
ProcessLifecycleOwner.get().lifecycle.addObserver(pinLocker)
ProcessLifecycleOwner.get().lifecycle.addObserver(webRtcPeerConnectionManager)
ProcessLifecycleOwner.get().lifecycle.addObserver(callManager)
// This should be done as early as possible
// initKnownEmojiHashSet(appContext)

8
vector/src/main/java/im/vector/app/core/di/ActiveSessionHolder.kt
View File

@ -18,7 +18,7 @@ package im.vector.app.core.di
import arrow.core.Option
import im.vector.app.ActiveSessionDataSource
import im.vector.app.features.call.webrtc.WebRtcPeerConnectionManager
import im.vector.app.features.call.webrtc.WebRtcCallManager
import im.vector.app.features.crypto.keysrequest.KeyRequestHandler
import im.vector.app.features.crypto.verification.IncomingVerificationRequestHandler
import im.vector.app.features.notifications.PushRuleTriggerListener
@ -35,7 +35,7 @@ class ActiveSessionHolder @Inject constructor(private val authenticationService:
private val sessionObservableStore: ActiveSessionDataSource,
private val keyRequestHandler: KeyRequestHandler,
private val incomingVerificationRequestHandler: IncomingVerificationRequestHandler,
private val webRtcPeerConnectionManager: WebRtcPeerConnectionManager,
private val callManager: WebRtcCallManager,
private val pushRuleTriggerListener: PushRuleTriggerListener,
private val sessionListener: SessionListener,
private val imageManager: ImageManager
@ -52,7 +52,7 @@ class ActiveSessionHolder @Inject constructor(private val authenticationService:
incomingVerificationRequestHandler.start(session)
session.addListener(sessionListener)
pushRuleTriggerListener.startWithSession(session)
session.callSignalingService().addCallListener(webRtcPeerConnectionManager)
session.callSignalingService().addCallListener(callManager)
imageManager.onSessionStarted(session)
}
@ -60,7 +60,7 @@ class ActiveSessionHolder @Inject constructor(private val authenticationService:
// Do some cleanup first
getSafeActiveSession()?.let {
Timber.w("clearActiveSession of ${it.myUserId}")
it.callSignalingService().removeCallListener(webRtcPeerConnectionManager)
it.callSignalingService().removeCallListener(callManager)
it.removeListener(sessionListener)
}

4
vector/src/main/java/im/vector/app/core/di/VectorComponent.kt
View File

@ -29,7 +29,7 @@ import im.vector.app.core.error.ErrorFormatter
import im.vector.app.core.pushers.PushersManager
import im.vector.app.core.utils.AssetReader
import im.vector.app.core.utils.DimensionConverter
import im.vector.app.features.call.webrtc.WebRtcPeerConnectionManager
import im.vector.app.features.call.webrtc.WebRtcCallManager
import im.vector.app.features.configuration.VectorConfiguration
import im.vector.app.features.crypto.keysrequest.KeyRequestHandler
import im.vector.app.features.crypto.verification.IncomingVerificationRequestHandler
@ -153,7 +153,7 @@ interface VectorComponent {
fun pinLocker(): PinLocker
fun webRtcPeerConnectionManager(): WebRtcPeerConnectionManager
fun webRtcPeerConnectionManager(): WebRtcCallManager
@Component.Factory
interface Factory {

12
vector/src/main/java/im/vector/app/core/services/CallService.kt
View File

@ -25,7 +25,7 @@ import android.view.KeyEvent
import androidx.core.content.ContextCompat
import androidx.media.session.MediaButtonReceiver
import im.vector.app.core.extensions.vectorComponent
import im.vector.app.features.call.webrtc.WebRtcPeerConnectionManager
import im.vector.app.features.call.webrtc.WebRtcCallManager
import im.vector.app.features.call.telecom.CallConnection
import im.vector.app.features.notifications.NotificationUtils
import timber.log.Timber
@ -38,7 +38,7 @@ class CallService : VectorService(), WiredHeadsetStateReceiver.HeadsetEventListe
private val connections = mutableMapOf<String, CallConnection>()
private lateinit var notificationUtils: NotificationUtils
private lateinit var webRtcPeerConnectionManager: WebRtcPeerConnectionManager
private lateinit var callManager: WebRtcCallManager
private var callRingPlayerIncoming: CallRingPlayerIncoming? = null
private var callRingPlayerOutgoing: CallRingPlayerOutgoing? = null
@ -53,7 +53,7 @@ class CallService : VectorService(), WiredHeadsetStateReceiver.HeadsetEventListe
override fun onMediaButtonEvent(mediaButtonEvent: Intent?): Boolean {
val keyEvent = mediaButtonEvent?.getParcelableExtra<KeyEvent>(Intent.EXTRA_KEY_EVENT) ?: return false
if (keyEvent.keyCode == KeyEvent.KEYCODE_HEADSETHOOK) {
webRtcPeerConnectionManager.headSetButtonTapped()
callManager.headSetButtonTapped()
return true
}
return false
@ -63,7 +63,7 @@ class CallService : VectorService(), WiredHeadsetStateReceiver.HeadsetEventListe
override fun onCreate() {
super.onCreate()
notificationUtils = vectorComponent().notificationUtils()
webRtcPeerConnectionManager = vectorComponent().webRtcPeerConnectionManager()
callManager = vectorComponent().webRtcPeerConnectionManager()
callRingPlayerIncoming = CallRingPlayerIncoming(applicationContext)
callRingPlayerOutgoing = CallRingPlayerOutgoing(applicationContext)
wiredHeadsetStateReceiver = WiredHeadsetStateReceiver.createAndRegister(this, this)
@ -375,11 +375,11 @@ class CallService : VectorService(), WiredHeadsetStateReceiver.HeadsetEventListe
override fun onHeadsetEvent(event: WiredHeadsetStateReceiver.HeadsetPlugEvent) {
Timber.v("## VOIP: onHeadsetEvent $event")
webRtcPeerConnectionManager.onWiredDeviceEvent(event)
callManager.onWiredDeviceEvent(event)
}
override fun onBTHeadsetEvent(event: BluetoothHeadsetReceiver.BTHeadsetPlugEvent) {
Timber.v("## VOIP: onBTHeadsetEvent $event")
webRtcPeerConnectionManager.onWirelessDeviceEvent(event)
callManager.onWirelessDeviceEvent(event)
}
}

12
vector/src/main/java/im/vector/app/core/ui/views/ActiveCallViewHolder.kt
View File

@ -20,7 +20,7 @@ import android.view.View
import androidx.cardview.widget.CardView
import androidx.core.view.isVisible
import im.vector.app.core.utils.DebouncedClickListener
import im.vector.app.features.call.webrtc.WebRtcPeerConnectionManager
import im.vector.app.features.call.webrtc.WebRtcCallManager
import org.matrix.android.sdk.api.session.call.CallState
import im.vector.app.features.call.utils.EglUtils
import org.matrix.android.sdk.api.session.call.MxCall
@ -35,7 +35,7 @@ class ActiveCallViewHolder {
private var activeCallPipInitialized = false
fun updateCall(activeCall: MxCall?, webRtcPeerConnectionManager: WebRtcPeerConnectionManager) {
fun updateCall(activeCall: MxCall?, callManager: WebRtcCallManager) {
val hasActiveCall = activeCall?.state is CallState.Connected
if (hasActiveCall) {
val isVideoCall = activeCall?.isVideoCall == true
@ -44,14 +44,14 @@ class ActiveCallViewHolder {
pipWrapper?.isVisible = isVideoCall
activeCallPiP?.isVisible = isVideoCall
activeCallPiP?.let {
webRtcPeerConnectionManager.attachViewRenderers(null, it, null)
callManager.attachViewRenderers(null, it, null)
}
} else {
activeCallView?.isVisible = false
activeCallPiP?.isVisible = false
pipWrapper?.isVisible = false
activeCallPiP?.let {
webRtcPeerConnectionManager.detachRenderers(listOf(it))
callManager.detachRenderers(listOf(it))
}
}
}
@ -82,9 +82,9 @@ class ActiveCallViewHolder {
)
}
fun unBind(webRtcPeerConnectionManager: WebRtcPeerConnectionManager) {
fun unBind(callManager: WebRtcCallManager) {
activeCallPiP?.let {
webRtcPeerConnectionManager.detachRenderers(listOf(it))
callManager.detachRenderers(listOf(it))
}
if (activeCallPipInitialized) {
activeCallPiP?.release()

12
vector/src/main/java/im/vector/app/features/call/SharedActiveCallViewModel.kt
View File

@ -18,12 +18,12 @@ package im.vector.app.features.call
import androidx.lifecycle.MutableLiveData
import androidx.lifecycle.ViewModel
import im.vector.app.features.call.webrtc.WebRtcPeerConnectionManager
import im.vector.app.features.call.webrtc.WebRtcCallManager
import org.matrix.android.sdk.api.session.call.MxCall
import javax.inject.Inject
class SharedActiveCallViewModel @Inject constructor(
private val webRtcPeerConnectionManager: WebRtcPeerConnectionManager
private val callManager: WebRtcCallManager
) : ViewModel() {
val activeCall: MutableLiveData<MxCall?> = MutableLiveData()
@ -37,7 +37,7 @@ class SharedActiveCallViewModel @Inject constructor(
}
}
private val listener = object : WebRtcPeerConnectionManager.CurrentCallListener {
private val listener = object : WebRtcCallManager.CurrentCallListener {
override fun onCurrentCallChange(call: MxCall?) {
activeCall.value?.removeListener(callStateListener)
activeCall.postValue(call)
@ -46,13 +46,13 @@ class SharedActiveCallViewModel @Inject constructor(
}
init {
activeCall.postValue(webRtcPeerConnectionManager.currentCall?.mxCall)
webRtcPeerConnectionManager.addCurrentCallListener(listener)
activeCall.postValue(callManager.currentCall?.mxCall)
callManager.addCurrentCallListener(listener)
}
override fun onCleared() {
activeCall.value?.removeListener(callStateListener)
webRtcPeerConnectionManager.removeCurrentCallListener(listener)
callManager.removeCurrentCallListener(listener)
super.onCleared()
}
}

14
vector/src/main/java/im/vector/app/features/call/VectorCallActivity.kt
View File

@ -53,7 +53,7 @@ import kotlinx.android.parcel.Parcelize
import kotlinx.android.synthetic.main.activity_call.*
import org.matrix.android.sdk.api.session.call.CallState
import im.vector.app.features.call.utils.EglUtils
import im.vector.app.features.call.webrtc.WebRtcPeerConnectionManager
import im.vector.app.features.call.webrtc.WebRtcCallManager
import org.matrix.android.sdk.api.session.call.MxCallDetail
import org.matrix.android.sdk.api.session.call.MxPeerConnectionState
import org.matrix.android.sdk.api.session.call.TurnServerResponse
@ -67,7 +67,7 @@ import javax.inject.Inject
@Parcelize
data class CallArgs(
val roomId: String,
val callId: String?,
val callId: String,
val participantUserId: String,
val isIncomingCall: Boolean,
val isVideoCall: Boolean
@ -87,7 +87,7 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis
private val callViewModel: VectorCallViewModel by viewModel()
private lateinit var callArgs: CallArgs
@Inject lateinit var peerConnectionManager: WebRtcPeerConnectionManager
@Inject lateinit var callManager: WebRtcCallManager
@Inject lateinit var viewModelFactory: VectorCallViewModel.Factory
@ -211,7 +211,7 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis
}
override fun onDestroy() {
peerConnectionManager.detachRenderers(listOf(pipRenderer, fullscreenRenderer))
callManager.detachRenderers(listOf(pipRenderer, fullscreenRenderer))
if (surfaceRenderersAreInitialized) {
pipRenderer.release()
fullscreenRenderer.release()
@ -276,7 +276,7 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis
callConnectingProgress.isVisible = true
}
// ensure all attached?
peerConnectionManager.attachViewRenderers(pipRenderer, fullscreenRenderer, null)
callManager.attachViewRenderers(pipRenderer, fullscreenRenderer, null)
}
is CallState.Terminated -> {
finish()
@ -326,7 +326,7 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis
pipRenderer.setEnableHardwareScaler(true /* enabled */)
fullscreenRenderer.setEnableHardwareScaler(true /* enabled */)
peerConnectionManager.attachViewRenderers(pipRenderer, fullscreenRenderer,
callManager.attachViewRenderers(pipRenderer, fullscreenRenderer,
intent.getStringExtra(EXTRA_MODE)?.takeIf { isFirstCreation() })
pipRenderer.setOnClickListener {
@ -382,7 +382,7 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis
}
fun newIntent(context: Context,
callId: String?,
callId: String,
roomId: String,
otherUserId: String,
isIncomingCall: Boolean,

167
vector/src/main/java/im/vector/app/features/call/VectorCallViewModel.kt
View File

@ -26,6 +26,8 @@ import com.squareup.inject.assisted.Assisted
import com.squareup.inject.assisted.AssistedInject
import im.vector.app.core.extensions.exhaustive
import im.vector.app.core.platform.VectorViewModel
import im.vector.app.features.call.webrtc.WebRtcCall
import im.vector.app.features.call.webrtc.WebRtcCallManager
import org.matrix.android.sdk.api.MatrixCallback
import org.matrix.android.sdk.api.session.Session
import org.matrix.android.sdk.api.session.call.CallState
@ -34,24 +36,41 @@ import org.matrix.android.sdk.api.session.call.MxPeerConnectionState
import org.matrix.android.sdk.api.session.call.TurnServerResponse
import org.matrix.android.sdk.api.util.MatrixItem
import org.matrix.android.sdk.api.util.toMatrixItem
import org.webrtc.PeerConnection
import java.util.Timer
import java.util.TimerTask
class VectorCallViewModel @AssistedInject constructor(
@Assisted initialState: VectorCallViewState,
@Assisted val args: CallArgs,
val session: Session,
val webRtcPeerConnectionManager: WebRtcPeerConnectionManager,
val callManager: WebRtcCallManager,
val proximityManager: CallProximityManager
) : VectorViewModel<VectorCallViewState, VectorCallViewActions, VectorCallViewEvents>(initialState) {
private var call: MxCall? = null
private var call: WebRtcCall? = null
private var connectionTimeoutTimer: Timer? = null
private var hasBeenConnectedOnce = false
private val callStateListener = object : MxCall.StateListener {
private val callListener = object : WebRtcCall.Listener {
override fun onCaptureStateChanged() {
setState {
copy(
isVideoCaptureInError = call?.videoCapturerIsInError ?: false,
isHD = call?.currentCaptureFormat() is CaptureFormat.HD
)
}
}
override fun onCameraChange() {
setState {
copy(
canSwitchCamera = call?.canSwitchCamera() ?: false,
isFrontCamera = call?.currentCameraType() == CameraType.FRONT
)
}
}
override fun onStateUpdate(call: MxCall) {
val callState = call.state
if (callState is CallState.Connected && callState.iceConnectionState == MxPeerConnectionState.CONNECTED) {
@ -87,7 +106,7 @@ class VectorCallViewModel @AssistedInject constructor(
}
}
private val currentCallListener = object : WebRtcPeerConnectionManager.CurrentCallListener {
private val currentCallListener = object : WebRtcCallManager.CurrentCallListener {
override fun onCurrentCallChange(call: MxCall?) {
// we need to check the state
if (call == null) {
@ -96,17 +115,8 @@ class VectorCallViewModel @AssistedInject constructor(
}
}
override fun onCaptureStateChanged() {
setState {
copy(
isVideoCaptureInError = webRtcPeerConnectionManager.capturerIsInError,
isHD = webRtcPeerConnectionManager.currentCaptureFormat() is CaptureFormat.HD
)
}
}
override fun onAudioDevicesChange() {
val currentSoundDevice = webRtcPeerConnectionManager.callAudioManager.getCurrentSoundDevice()
val currentSoundDevice = callManager.callAudioManager.getCurrentSoundDevice()
if (currentSoundDevice == CallAudioManager.SoundDevice.PHONE) {
proximityManager.start()
} else {
@ -115,94 +125,77 @@ class VectorCallViewModel @AssistedInject constructor(
setState {
copy(
availableSoundDevices = webRtcPeerConnectionManager.callAudioManager.getAvailableSoundDevices(),
availableSoundDevices = callManager.callAudioManager.getAvailableSoundDevices(),
soundDevice = currentSoundDevice
)
}
}
override fun onCameraChange() {
}
init {
val webRtcCall = callManager.getCallById(initialState.callId)
if (webRtcCall == null) {
setState {
copy(callState = Fail(IllegalArgumentException("No call")))
}
} else {
call = webRtcCall
callManager.addCurrentCallListener(currentCallListener)
val item: MatrixItem? = session.getUser(webRtcCall.mxCall.opponentUserId)?.toMatrixItem()
webRtcCall.addListener(callListener)
val currentSoundDevice = callManager.callAudioManager.getCurrentSoundDevice()
if (currentSoundDevice == CallAudioManager.SoundDevice.PHONE) {
proximityManager.start()
}
setState {
copy(
canSwitchCamera = webRtcPeerConnectionManager.canSwitchCamera(),
isFrontCamera = webRtcPeerConnectionManager.currentCameraType() == CameraType.FRONT
isVideoCall = webRtcCall.mxCall.isVideoCall,
callState = Success(webRtcCall.mxCall.state),
otherUserMatrixItem = item?.let { Success(it) } ?: Uninitialized,
soundDevice = currentSoundDevice,
availableSoundDevices = callManager.callAudioManager.getAvailableSoundDevices(),
isFrontCamera = callManager.currentCameraType() == CameraType.FRONT,
canSwitchCamera = callManager.canSwitchCamera(),
isHD = webRtcCall.mxCall.isVideoCall && callManager.currentCaptureFormat() is CaptureFormat.HD
)
}
}
}
init {
initialState.callId?.let {
webRtcPeerConnectionManager.addCurrentCallListener(currentCallListener)
session.callSignalingService().getCallWithId(it)?.let { mxCall ->
this.call = mxCall
mxCall.opponentUserId
val item: MatrixItem? = session.getUser(mxCall.opponentUserId)?.toMatrixItem()
mxCall.addListener(callStateListener)
val currentSoundDevice = webRtcPeerConnectionManager.callAudioManager.getCurrentSoundDevice()
if (currentSoundDevice == CallAudioManager.SoundDevice.PHONE) {
proximityManager.start()
}
setState {
copy(
isVideoCall = mxCall.isVideoCall,
callState = Success(mxCall.state),
otherUserMatrixItem = item?.let { Success(it) } ?: Uninitialized,
soundDevice = currentSoundDevice,
availableSoundDevices = webRtcPeerConnectionManager.callAudioManager.getAvailableSoundDevices(),
isFrontCamera = webRtcPeerConnectionManager.currentCameraType() == CameraType.FRONT,
canSwitchCamera = webRtcPeerConnectionManager.canSwitchCamera(),
isHD = mxCall.isVideoCall && webRtcPeerConnectionManager.currentCaptureFormat() is CaptureFormat.HD
)
}
} ?: run {
setState {
copy(
callState = Fail(IllegalArgumentException("No call"))
)
}
}
}
}
override fun onCleared() {
// session.callService().removeCallListener(callServiceListener)
webRtcPeerConnectionManager.removeCurrentCallListener(currentCallListener)
this.call?.removeListener(callStateListener)
callManager.removeCurrentCallListener(currentCallListener)
call?.removeListener(callListener)
proximityManager.stop()
super.onCleared()
}
override fun handle(action: VectorCallViewActions) = withState { state ->
when (action) {
VectorCallViewActions.EndCall -> webRtcPeerConnectionManager.endCall()
VectorCallViewActions.AcceptCall -> {
VectorCallViewActions.EndCall -> call?.endCall()
VectorCallViewActions.AcceptCall -> {
setState {
copy(callState = Loading())
}
webRtcPeerConnectionManager.acceptIncomingCall()
call?.acceptIncomingCall()
}
VectorCallViewActions.DeclineCall -> {
VectorCallViewActions.DeclineCall -> {
setState {
copy(callState = Loading())
}
webRtcPeerConnectionManager.endCall()
call?.endCall()
}
VectorCallViewActions.ToggleMute -> {
VectorCallViewActions.ToggleMute -> {
val muted = state.isAudioMuted
webRtcPeerConnectionManager.muteCall(!muted)
call?.muteCall(!muted)
setState {
copy(isAudioMuted = !muted)
}
}
VectorCallViewActions.ToggleVideo -> {
VectorCallViewActions.ToggleVideo -> {
if (state.isVideoCall) {
val videoEnabled = state.isVideoEnabled
webRtcPeerConnectionManager.enableVideo(!videoEnabled)
call?.enableVideo(!videoEnabled)
setState {
copy(isVideoEnabled = !videoEnabled)
}
@ -210,14 +203,14 @@ class VectorCallViewModel @AssistedInject constructor(
Unit
}
is VectorCallViewActions.ChangeAudioDevice -> {
webRtcPeerConnectionManager.callAudioManager.setCurrentSoundDevice(action.device)
callManager.callAudioManager.setCurrentSoundDevice(action.device)
setState {
copy(
soundDevice = webRtcPeerConnectionManager.callAudioManager.getCurrentSoundDevice()
soundDevice = callManager.callAudioManager.getCurrentSoundDevice()
)
}
}
VectorCallViewActions.SwitchSoundDevice -> {
VectorCallViewActions.SwitchSoundDevice -> {
_viewEvents.post(
VectorCallViewEvents.ShowSoundDeviceChooser(state.availableSoundDevices, state.soundDevice)
)
@ -225,45 +218,35 @@ class VectorCallViewModel @AssistedInject constructor(
VectorCallViewActions.HeadSetButtonPressed -> {
if (state.callState.invoke() is CallState.LocalRinging) {
// accept call
webRtcPeerConnectionManager.acceptIncomingCall()
call?.acceptIncomingCall()
}
if (state.callState.invoke() is CallState.Connected) {
// end call?
webRtcPeerConnectionManager.endCall()
call?.endCall()
}
Unit
}
VectorCallViewActions.ToggleCamera -> {
webRtcPeerConnectionManager.switchCamera()
VectorCallViewActions.ToggleCamera -> {
call?.switchCamera()
}
VectorCallViewActions.ToggleHDSD -> {
VectorCallViewActions.ToggleHDSD -> {
if (!state.isVideoCall) return@withState
webRtcPeerConnectionManager.setCaptureFormat(if (state.isHD) CaptureFormat.SD else CaptureFormat.HD)
call?.setCaptureFormat(if (state.isHD) CaptureFormat.SD else CaptureFormat.HD)
}
}.exhaustive
}
@AssistedInject.Factory
interface Factory {
fun create(initialState: VectorCallViewState, args: CallArgs): VectorCallViewModel
fun create(initialState: VectorCallViewState): VectorCallViewModel
}
companion object : MvRxViewModelFactory<VectorCallViewModel, VectorCallViewState> {
@JvmStatic
override fun create(viewModelContext: ViewModelContext, state: VectorCallViewState): VectorCallViewModel? {
override fun create(viewModelContext: ViewModelContext, state: VectorCallViewState): VectorCallViewModel {
val callActivity: VectorCallActivity = viewModelContext.activity()
val callArgs: CallArgs = viewModelContext.args()
return callActivity.viewModelFactory.create(state, callArgs)
}
override fun initialState(viewModelContext: ViewModelContext): VectorCallViewState? {
val args: CallArgs = viewModelContext.args()
return VectorCallViewState(
callId = args.callId,
roomId = args.roomId,
isVideoCall = args.isVideoCall
)
return callActivity.viewModelFactory.create(state)
}
}
}

15
vector/src/main/java/im/vector/app/features/call/VectorCallViewState.kt
View File

@ -23,8 +23,8 @@ import org.matrix.android.sdk.api.session.call.CallState
import org.matrix.android.sdk.api.util.MatrixItem
data class VectorCallViewState(
val callId: String? = null,
val roomId: String = "",
val callId: String,
val roomId: String,
val isVideoCall: Boolean,
val isAudioMuted: Boolean = false,
val isVideoEnabled: Boolean = true,
@ -36,4 +36,13 @@ data class VectorCallViewState(
val availableSoundDevices: List<CallAudioManager.SoundDevice> = emptyList(),
val otherUserMatrixItem: Async<MatrixItem> = Uninitialized,
val callState: Async<CallState> = Uninitialized
) : MvRxState
) : MvRxState {
constructor(callArgs: CallArgs): this(
callId = callArgs.callId,
roomId = callArgs.roomId,
isVideoCall = callArgs.isVideoCall
)
}

6
vector/src/main/java/im/vector/app/features/call/service/CallHeadsUpActionReceiver.kt
View File

@ -20,7 +20,7 @@ import android.content.BroadcastReceiver
import android.content.Context
import android.content.Intent
import im.vector.app.core.di.HasVectorInjector
import im.vector.app.features.call.webrtc.WebRtcPeerConnectionManager
import im.vector.app.features.call.webrtc.WebRtcCallManager
import timber.log.Timber
class CallHeadsUpActionReceiver : BroadcastReceiver() {
@ -48,9 +48,9 @@ class CallHeadsUpActionReceiver : BroadcastReceiver() {
// context.stopService(Intent(context, CallHeadsUpService::class.java))
}
private fun onCallRejectClicked(peerConnectionManager: WebRtcPeerConnectionManager) {
private fun onCallRejectClicked(callManager: WebRtcCallManager) {
Timber.d("onCallRejectClicked")
peerConnectionManager.endCall()
callManager.endCall()
}
// private fun onCallAnswerClicked(context: Context) {

4
vector/src/main/java/im/vector/app/features/call/telecom/CallConnection.kt
View File

@ -22,7 +22,7 @@ import android.telecom.Connection
import android.telecom.DisconnectCause
import androidx.annotation.RequiresApi
import im.vector.app.features.call.VectorCallViewModel
import im.vector.app.features.call.webrtc.WebRtcPeerConnectionManager
import im.vector.app.features.call.webrtc.WebRtcCallManager
import timber.log.Timber
import javax.inject.Inject
@ -32,7 +32,7 @@ import javax.inject.Inject
val callId: String
) : Connection() {
@Inject lateinit var peerConnectionManager: WebRtcPeerConnectionManager
@Inject lateinit var callManager: WebRtcCallManager
@Inject lateinit var callViewModel: VectorCallViewModel
init {

10
vector/src/main/java/im/vector/app/features/call/webrtc/PeerConnectionObserver.kt
View File

@ -17,7 +17,6 @@
package im.vector.app.features.call.webrtc
import im.vector.app.features.call.CallAudioManager
import kotlinx.coroutines.GlobalScope
import org.matrix.android.sdk.api.session.call.CallState
import org.matrix.android.sdk.api.session.call.MxPeerConnectionState
import org.webrtc.DataChannel
@ -84,7 +83,7 @@ class PeerConnectionObserver(private val webRtcCall: WebRtcCall,
override fun onIceCandidate(iceCandidate: IceCandidate) {
Timber.v("## VOIP StreamObserver onIceCandidate: $iceCandidate")
webRtcCall.iceCandidateSource.onNext(iceCandidate)
webRtcCall.onIceCandidate(iceCandidate)
}
override fun onDataChannel(dc: DataChannel) {
@ -153,7 +152,6 @@ class PeerConnectionObserver(private val webRtcCall: WebRtcCall,
override fun onAddStream(stream: MediaStream) {
Timber.v("## VOIP StreamObserver onAddStream: $stream")
webRtcCall.onAddStream(stream)
}
override fun onRemoveStream(stream: MediaStream) {
@ -175,11 +173,7 @@ class PeerConnectionObserver(private val webRtcCall: WebRtcCall,
override fun onRenegotiationNeeded() {
Timber.v("## VOIP StreamObserver onRenegotiationNeeded")
if (webRtcCall.mxCall.state != CallState.CreateOffer && webRtcCall.mxCall.opponentVersion == 0) {
Timber.v("Opponent does not support renegotiation: ignoring onRenegotiationNeeded event")
return
}
webRtcCall.sendSpdOffer()
webRtcCall.onRenegationNeeded()
}
/**

282
vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCall.kt
View File

@ -37,7 +37,6 @@ import io.reactivex.subjects.PublishSubject
import io.reactivex.subjects.ReplaySubject
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.GlobalScope
import kotlinx.coroutines.asCoroutineDispatcher
import kotlinx.coroutines.delay
import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
@ -49,6 +48,7 @@ import org.matrix.android.sdk.api.session.call.MxCall
import org.matrix.android.sdk.api.session.call.TurnServerResponse
import org.matrix.android.sdk.api.session.room.model.call.CallAnswerContent
import org.matrix.android.sdk.api.session.room.model.call.CallCandidatesContent
import org.matrix.android.sdk.api.session.room.model.call.CallHangupContent
import org.matrix.android.sdk.api.session.room.model.call.CallInviteContent
import org.matrix.android.sdk.api.session.room.model.call.CallNegotiateContent
import org.matrix.android.sdk.api.session.room.model.call.SdpType
@ -72,9 +72,9 @@ import org.webrtc.VideoSource
import org.webrtc.VideoTrack
import timber.log.Timber
import java.lang.ref.WeakReference
import java.util.concurrent.Executor
import java.util.concurrent.TimeUnit
import javax.inject.Provider
import kotlin.coroutines.CoroutineContext
private const val STREAM_ID = "ARDAMS"
private const val AUDIO_TRACK_ID = "ARDAMSa0"
@ -85,29 +85,44 @@ class WebRtcCall(val mxCall: MxCall,
private val callAudioManager: CallAudioManager,
private val rootEglBase: EglBase?,
private val context: Context,
private val session: Session,
private val executor: Executor,
private val peerConnectionFactoryProvider: Provider<PeerConnectionFactory?>) {
private val dispatcher: CoroutineContext,
private val sessionProvider: Provider<Session?>,
private val peerConnectionFactoryProvider: Provider<PeerConnectionFactory?>,
private val onCallEnded: (WebRtcCall) -> Unit): MxCall.StateListener {
private val dispatcher = executor.asCoroutineDispatcher()
interface Listener: MxCall.StateListener {
fun onCaptureStateChanged() {}
fun onCameraChange() {}
}
var peerConnection: PeerConnection? = null
var localAudioSource: AudioSource? = null
var localAudioTrack: AudioTrack? = null
var localVideoSource: VideoSource? = null
var localVideoTrack: VideoTrack? = null
var remoteVideoTrack: VideoTrack? = null
private val listeners = ArrayList<Listener>()
fun addListener(listener: Listener) {
listeners.add(listener)
}
fun removeListener(listener: Listener) {
listeners.remove(listener)
}
val callId = mxCall.callId
private var peerConnection: PeerConnection? = null
private var localAudioSource: AudioSource? = null
private var localAudioTrack: AudioTrack? = null
private var localVideoSource: VideoSource? = null
private var localVideoTrack: VideoTrack? = null
private var remoteVideoTrack: VideoTrack? = null
// Perfect negotiation state: https://www.w3.org/TR/webrtc/#perfect-negotiation-example
var makingOffer: Boolean = false
var ignoreOffer: Boolean = false
private var makingOffer: Boolean = false
private var ignoreOffer: Boolean = false
private var videoCapturer: CameraVideoCapturer? = null
private val availableCamera = ArrayList<CameraProxy>()
private var cameraInUse: CameraProxy? = null
private var currentCaptureFormat: CaptureFormat = CaptureFormat.HD
private var capturerIsInError = false
private var cameraAvailabilityCallback: CameraManager.AvailabilityCallback? = null
// Mute status
@ -117,10 +132,17 @@ class WebRtcCall(val mxCall: MxCall,
var offerSdp: CallInviteContent.Offer? = null
var videoCapturerIsInError = false
set(value) {
field = value
listeners.forEach {
tryOrNull { it.onCaptureStateChanged() }
}
}
private var localSurfaceRenderers: MutableList<WeakReference<SurfaceViewRenderer>> = ArrayList()
private var remoteSurfaceRenderers: MutableList<WeakReference<SurfaceViewRenderer>> = ArrayList()
val iceCandidateSource: PublishSubject<IceCandidate> = PublishSubject.create()
private val iceCandidateSource: PublishSubject<IceCandidate> = PublishSubject.create()
private val iceCandidateDisposable = iceCandidateSource
.buffer(300, TimeUnit.MILLISECONDS)
.subscribe {
@ -132,60 +154,51 @@ class WebRtcCall(val mxCall: MxCall,
}
}
var remoteCandidateSource: ReplaySubject<IceCandidate> = ReplaySubject.create()
var remoteIceCandidateDisposable: Disposable? = null
private val remoteCandidateSource: ReplaySubject<IceCandidate> = ReplaySubject.create()
private var remoteIceCandidateDisposable: Disposable? = null
private fun createLocalStream() {
val peerConnectionFactory = peerConnectionFactoryProvider.get() ?: return
Timber.v("Create local stream for call ${mxCall.callId}")
configureAudioTrack(peerConnectionFactory)
// add video track if needed
if (mxCall.isVideoCall) {
configureVideoTrack(peerConnectionFactory)
}
updateMuteStatus()
init {
mxCall.addListener(this)
}
private fun configureAudioTrack(peerConnectionFactory: PeerConnectionFactory) {
val audioSource = peerConnectionFactory.createAudioSource(DEFAULT_AUDIO_CONSTRAINTS)
val audioTrack = peerConnectionFactory.createAudioTrack(AUDIO_TRACK_ID, audioSource)
audioTrack.setEnabled(true)
Timber.v("Add audio track $AUDIO_TRACK_ID to call ${mxCall.callId}")
peerConnection?.addTrack(audioTrack, listOf(STREAM_ID))
localAudioSource = audioSource
localAudioTrack = audioTrack
}
fun onIceCandidate(iceCandidate: IceCandidate) = iceCandidateSource.onNext(iceCandidate)
fun sendSpdOffer() = GlobalScope.launch(dispatcher) {
val constraints = MediaConstraints()
// These are deprecated options
fun onRenegationNeeded() {
GlobalScope.launch(dispatcher) {
if (mxCall.state != CallState.CreateOffer && mxCall.opponentVersion == 0) {
Timber.v("Opponent does not support renegotiation: ignoring onRenegotiationNeeded event")
return@launch
}
val constraints = MediaConstraints()
// These are deprecated options
// constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true"))
// constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveVideo", if (currentCall?.mxCall?.isVideoCall == true) "true" else "false"))
val peerConnection = peerConnection ?: return@launch
Timber.v("## VOIP creating offer...")
makingOffer = true
try {
val sessionDescription = peerConnection.awaitCreateOffer(constraints) ?: return@launch
peerConnection.awaitSetLocalDescription(sessionDescription)
if (peerConnection.iceGatheringState() == PeerConnection.IceGatheringState.GATHERING) {
// Allow a short time for initial candidates to be gathered
delay(200)
val peerConnection = peerConnection ?: return@launch
Timber.v("## VOIP creating offer...")
makingOffer = true
try {
val sessionDescription = peerConnection.awaitCreateOffer(constraints) ?: return@launch
peerConnection.awaitSetLocalDescription(sessionDescription)
if (peerConnection.iceGatheringState() == PeerConnection.IceGatheringState.GATHERING) {
// Allow a short time for initial candidates to be gathered
delay(200)
}
if (mxCall.state == CallState.Terminated) {
return@launch
}
if (mxCall.state == CallState.CreateOffer) {
// send offer to peer
mxCall.offerSdp(sessionDescription.description)
} else {
mxCall.negotiate(sessionDescription.description)
}
} catch (failure: Throwable) {
// Need to handle error properly.
Timber.v("Failure while creating offer")
} finally {
makingOffer = false
}
if (mxCall.state == CallState.Terminated) {
return@launch
}
if (mxCall.state == CallState.CreateOffer) {
// send offer to peer
mxCall.offerSdp(sessionDescription.description)
} else {
mxCall.negotiate(sessionDescription.description)
}
} catch (failure: Throwable) {
// Need to handle error properly.
Timber.v("Failure while creating offer")
} finally {
makingOffer = false
}
}
@ -223,7 +236,8 @@ class WebRtcCall(val mxCall: MxCall,
mxCall
.takeIf { it.state is CallState.Connected }
?.let { mxCall ->
val name = session.getUser(mxCall.opponentUserId)?.getBestName()
val session = sessionProvider.get()
val name = session?.getUser(mxCall.opponentUserId)?.getBestName()
?: mxCall.roomId
// Start background service with notification
CallService.onPendingCall(
@ -231,7 +245,7 @@ class WebRtcCall(val mxCall: MxCall,
isVideo = mxCall.isVideoCall,
roomName = name,
roomId = mxCall.roomId,
matrixId = session.myUserId,
matrixId = session?.myUserId ?:"",
callId = mxCall.callId)
}
@ -255,9 +269,12 @@ class WebRtcCall(val mxCall: MxCall,
}
}
fun acceptIncomingCall() = GlobalScope.launch {
if (mxCall.state == CallState.LocalRinging) {
internalAcceptIncomingCall()
fun acceptIncomingCall() {
GlobalScope.launch {
Timber.v("## VOIP acceptIncomingCall from state ${mxCall.state}")
if (mxCall.state == CallState.LocalRinging) {
internalAcceptIncomingCall()
}
}
}
@ -289,14 +306,15 @@ class WebRtcCall(val mxCall: MxCall,
.takeIf { it.state is CallState.Connected }
?.let { mxCall ->
// Start background service with notification
val name = session.getUser(mxCall.opponentUserId)?.getBestName()
val session = sessionProvider.get()
val name = session?.getUser(mxCall.opponentUserId)?.getBestName()
?: mxCall.opponentUserId
CallService.onOnGoingCallBackground(
context = context,
isVideo = mxCall.isVideoCall,
roomName = name,
roomId = mxCall.roomId,
matrixId = session.myUserId ,
matrixId = session?.myUserId ?: "",
callId = mxCall.callId
)
}
@ -325,14 +343,15 @@ class WebRtcCall(val mxCall: MxCall,
val turnServerResponse = getTurnServer()
// Update service state
withContext(Dispatchers.Main) {
val name = session.getUser(mxCall.opponentUserId)?.getBestName()
val session = sessionProvider.get()
val name = session?.getUser(mxCall.opponentUserId)?.getBestName()
?: mxCall.roomId
CallService.onPendingCall(
context = context,
isVideo = mxCall.isVideoCall,
roomName = name,
roomId = mxCall.roomId,
matrixId = session.myUserId,
matrixId = session?.myUserId ?: "",
callId = mxCall.callId
)
}
@ -393,13 +412,33 @@ class WebRtcCall(val mxCall: MxCall,
private suspend fun getTurnServer(): TurnServerResponse? {
return tryOrNull {
awaitCallback {
session.callSignalingService().getTurnServer(it)
sessionProvider.get()?.callSignalingService()?.getTurnServer(it)
}
}
}
private fun createLocalStream() {
val peerConnectionFactory = peerConnectionFactoryProvider.get() ?: return
Timber.v("Create local stream for call ${mxCall.callId}")
configureAudioTrack(peerConnectionFactory)
// add video track if needed
if (mxCall.isVideoCall) {
configureVideoTrack(peerConnectionFactory)
}
updateMuteStatus()
}
private fun configureAudioTrack(peerConnectionFactory: PeerConnectionFactory) {
val audioSource = peerConnectionFactory.createAudioSource(DEFAULT_AUDIO_CONSTRAINTS)
val audioTrack = peerConnectionFactory.createAudioTrack(AUDIO_TRACK_ID, audioSource)
audioTrack.setEnabled(true)
Timber.v("Add audio track $AUDIO_TRACK_ID to call ${mxCall.callId}")
peerConnection?.addTrack(audioTrack, listOf(STREAM_ID))
localAudioSource = audioSource
localAudioTrack = audioTrack
}
private fun configureVideoTrack(peerConnectionFactory: PeerConnectionFactory) {
availableCamera.clear()
val cameraIterator = if (Camera2Enumerator.isSupported(context)) {
Camera2Enumerator(context)
} else {
@ -426,14 +465,14 @@ class WebRtcCall(val mxCall: MxCall,
val videoCapturer = cameraIterator.createCapturer(camera.name, object : CameraEventsHandlerAdapter() {
override fun onFirstFrameAvailable() {
super.onFirstFrameAvailable()
capturerIsInError = false
videoCapturerIsInError = false
}
override fun onCameraClosed() {
super.onCameraClosed()
// This could happen if you open the camera app in chat
// We then register in order to restart capture as soon as the camera is available again
capturerIsInError = true
videoCapturerIsInError = true
val cameraManager = context.getSystemService<CameraManager>()
cameraAvailabilityCallback = object : CameraManager.AvailabilityCallback() {
override fun onCameraAvailable(cameraId: String) {
@ -466,12 +505,10 @@ class WebRtcCall(val mxCall: MxCall,
}
fun setCaptureFormat(format: CaptureFormat) {
Timber.v("## VOIP setCaptureFormat $format")
executor.execute {
// videoCapturer?.stopCapture()
GlobalScope.launch(dispatcher) {
Timber.v("## VOIP setCaptureFormat $format")
videoCapturer?.changeCaptureFormat(format.width, format.height, format.fps)
currentCaptureFormat = format
//currentCallsListeners.forEach { tryOrNull { it.onCaptureStateChanged() } }
}
}
@ -543,6 +580,10 @@ class WebRtcCall(val mxCall: MxCall,
localSurfaceRenderers.forEach {
it.get()?.setMirror(isFrontCamera)
}
listeners.forEach {
tryOrNull { it.onCameraChange() }
}
}
override fun onCameraSwitchError(errorDescription: String?) {
@ -577,7 +618,8 @@ class WebRtcCall(val mxCall: MxCall,
return currentCaptureFormat
}
fun release() {
private fun release() {
mxCall.removeListener(this)
videoCapturer?.stopCapture()
videoCapturer?.dispose()
videoCapturer = null
@ -591,21 +633,22 @@ class WebRtcCall(val mxCall: MxCall,
localAudioTrack = null
localVideoSource = null
localVideoTrack = null
cameraAvailabilityCallback = null
}
fun onAddStream(stream: MediaStream) {
executor.execute {
GlobalScope.launch(dispatcher) {
// reportError("Weird-looking stream: " + stream);
if (stream.audioTracks.size > 1 || stream.videoTracks.size > 1) {
Timber.e("## VOIP StreamObserver weird looking stream: $stream")
// TODO maybe do something more??
mxCall.hangUp()
return@execute
return@launch
}
if (stream.videoTracks.size == 1) {
val remoteVideoTrack = stream.videoTracks.first()
remoteVideoTrack.setEnabled(true)
this.remoteVideoTrack = remoteVideoTrack
this@WebRtcCall.remoteVideoTrack = remoteVideoTrack
// sink to renderer if attached
remoteSurfaceRenderers.forEach { it.get()?.let { remoteVideoTrack.addSink(it) } }
}
@ -613,7 +656,7 @@ class WebRtcCall(val mxCall: MxCall,
}
fun onRemoveStream() {
executor.execute {
GlobalScope.launch(dispatcher) {
remoteSurfaceRenderers
.mapNotNull { it.get() }
.forEach { remoteVideoTrack?.removeSink(it) }
@ -621,26 +664,31 @@ class WebRtcCall(val mxCall: MxCall,
}
}
fun endCall(originatedByMe: Boolean) {
fun endCall(originatedByMe: Boolean = true, reason: CallHangupContent.Reason? = null) {
mxCall.state = CallState.Terminated
//Close tracks ASAP
localVideoTrack?.setEnabled(false)
localVideoTrack?.setEnabled(false)
cameraAvailabilityCallback?.let { cameraAvailabilityCallback ->
val cameraManager = context.getSystemService<CameraManager>()!!
cameraManager.unregisterAvailabilityCallback(cameraAvailabilityCallback)
}
release()
onCallEnded(this)
if (originatedByMe) {
// send hang up event
mxCall.hangUp()
if (mxCall.state is CallState.Connected) {
mxCall.hangUp(reason)
} else {
mxCall.reject()
}
}
}
// Call listener
fun onCallIceCandidateReceived(iceCandidatesContent: CallCandidatesContent) {
executor.execute {
GlobalScope.launch(dispatcher) {
iceCandidatesContent.candidates.forEach {
Timber.v("## VOIP onCallIceCandidateReceived for call ${mxCall.callId} sdp: ${it.candidate}")
val iceCandidate = IceCandidate(it.sdpMid, it.sdpMLineIndex, it.candidate)
@ -665,43 +713,49 @@ class WebRtcCall(val mxCall: MxCall,
}
fun onCallNegotiateReceived(callNegotiateContent: CallNegotiateContent) {
val description = callNegotiateContent.description
val type = description?.type
val sdpText = description?.sdp
if (type == null || sdpText == null) {
Timber.i("Ignoring invalid m.call.negotiate event");
return;
}
val peerConnection = peerConnection ?: return
// Politeness always follows the direction of the call: in a glare situation,
// we pick either the inbound or outbound call, so one side will always be
// inbound and one outbound
val polite = !mxCall.isOutgoing
// Here we follow the perfect negotiation logic from
// https://developer.mozilla.org/en-US/docs/Web/API/WebRTC_API/Perfect_negotiation
val offerCollision = description.type == SdpType.OFFER
&& (makingOffer || peerConnection.signalingState() != PeerConnection.SignalingState.STABLE)
ignoreOffer = !polite && offerCollision
if (ignoreOffer) {
Timber.i("Ignoring colliding negotiate event because we're impolite")
return
}
GlobalScope.launch(dispatcher) {
val description = callNegotiateContent.description
val type = description?.type
val sdpText = description?.sdp
if (type == null || sdpText == null) {
Timber.i("Ignoring invalid m.call.negotiate event");
return@launch
}
val peerConnection = peerConnection ?: return@launch
// Politeness always follows the direction of the call: in a glare situation,
// we pick either the inbound or outbound call, so one side will always be
// inbound and one outbound
val polite = !mxCall.isOutgoing
// Here we follow the perfect negotiation logic from
// https://developer.mozilla.org/en-US/docs/Web/API/WebRTC_API/Perfect_negotiation
val offerCollision = description.type == SdpType.OFFER
&& (makingOffer || peerConnection.signalingState() != PeerConnection.SignalingState.STABLE)
ignoreOffer = !polite && offerCollision
if (ignoreOffer) {
Timber.i("Ignoring colliding negotiate event because we're impolite")
return@launch
}
try {
val sdp = SessionDescription(type.asWebRTC(), sdpText)
peerConnection.awaitSetRemoteDescription(sdp)
if (type == SdpType.OFFER) {
createAnswer()?.also {
mxCall.negotiate(sdpText)
}
createAnswer()
mxCall.negotiate(sdpText)
}
} catch (failure: Throwable) {
Timber.e(failure, "Failed to complete negotiation")
}
}
}
// MxCall.StateListener
override fun onStateUpdate(call: MxCall) {
listeners.forEach {
tryOrNull { it.onStateUpdate(call) }
}
}
}
private fun MutableList<WeakReference<SurfaceViewRenderer>>.addIfNeeded(renderer: SurfaceViewRenderer?) {

187
vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcPeerConnectionManager.kt → vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCallManager.kt
View File

@ -29,8 +29,6 @@ import im.vector.app.features.call.CameraType
import im.vector.app.features.call.CaptureFormat
import im.vector.app.features.call.VectorCallActivity
import im.vector.app.features.call.utils.EglUtils
import im.vector.app.features.call.utils.awaitCreateAnswer
import im.vector.app.features.call.utils.awaitSetLocalDescription
import im.vector.app.push.fcm.FcmHelper
import kotlinx.coroutines.asCoroutineDispatcher
import org.matrix.android.sdk.api.extensions.orFalse
@ -39,7 +37,6 @@ import org.matrix.android.sdk.api.session.Session
import org.matrix.android.sdk.api.session.call.CallListener
import org.matrix.android.sdk.api.session.call.CallState
import org.matrix.android.sdk.api.session.call.MxCall
import org.matrix.android.sdk.api.session.call.TurnServerResponse
import org.matrix.android.sdk.api.session.room.model.call.CallAnswerContent
import org.matrix.android.sdk.api.session.room.model.call.CallCandidatesContent
import org.matrix.android.sdk.api.session.room.model.call.CallHangupContent
@ -47,18 +44,13 @@ import org.matrix.android.sdk.api.session.room.model.call.CallInviteContent
import org.matrix.android.sdk.api.session.room.model.call.CallNegotiateContent
import org.matrix.android.sdk.api.session.room.model.call.CallRejectContent
import org.matrix.android.sdk.api.session.room.model.call.CallSelectAnswerContent
import org.matrix.android.sdk.internal.util.awaitCallback
import org.webrtc.DefaultVideoDecoderFactory
import org.webrtc.DefaultVideoEncoderFactory
import org.webrtc.MediaConstraints
import org.webrtc.PeerConnectionFactory
import org.webrtc.SessionDescription
import org.webrtc.SurfaceViewRenderer
import timber.log.Timber
import java.lang.ref.WeakReference
import java.util.concurrent.Executors
import javax.inject.Inject
import javax.inject.Provider
import javax.inject.Singleton
/**
@ -66,7 +58,7 @@ import javax.inject.Singleton
* Use app context
*/
@Singleton
class WebRtcPeerConnectionManager @Inject constructor(
class WebRtcCallManager @Inject constructor(
private val context: Context,
private val activeSessionDataSource: ActiveSessionDataSource
) : CallListener, LifecycleObserver {
@ -81,6 +73,14 @@ class WebRtcPeerConnectionManager @Inject constructor(
fun onCameraChange() {}
}
var capturerIsInError = false
set(value) {
field = value
currentCallsListeners.forEach {
tryOrNull { it.onCaptureStateChanged() }
}
}
private val currentCallsListeners = emptyList<CurrentCallListener>().toMutableList()
fun addCurrentCallListener(listener: CurrentCallListener) {
currentCallsListeners.add(listener)
@ -90,7 +90,7 @@ class WebRtcPeerConnectionManager @Inject constructor(
currentCallsListeners.remove(listener)
}
val callAudioManager = CallAudioManager(context.applicationContext) {
val callAudioManager = CallAudioManager(context) {
currentCallsListeners.forEach {
tryOrNull { it.onAudioDevicesChange() }
}
@ -104,34 +104,6 @@ class WebRtcPeerConnectionManager @Inject constructor(
private var isInBackground: Boolean = true
var capturerIsInError = false
set(value) {
field = value
currentCallsListeners.forEach {
tryOrNull { it.onCaptureStateChanged() }
}
}
var localSurfaceRenderers: MutableList<WeakReference<SurfaceViewRenderer>> = ArrayList()
var remoteSurfaceRenderers: MutableList<WeakReference<SurfaceViewRenderer>> = ArrayList()
private fun MutableList<WeakReference<SurfaceViewRenderer>>.addIfNeeded(renderer: SurfaceViewRenderer?) {
if (renderer == null) return
val exists = any {
it.get() == renderer
}
if (!exists) {
add(WeakReference(renderer))
}
}
private fun MutableList<WeakReference<SurfaceViewRenderer>>.removeIfNeeded(renderer: SurfaceViewRenderer?) {
if (renderer == null) return
removeAll {
it.get() == renderer
}
}
@OnLifecycleEvent(Lifecycle.Event.ON_RESUME)
fun entersForeground() {
isInBackground = false
@ -150,6 +122,12 @@ class WebRtcPeerConnectionManager @Inject constructor(
}
}
private val callsByCallId = HashMap<String, WebRtcCall>()
fun getCallById(callId: String): WebRtcCall? {
return callsByCallId[callId]
}
fun headSetButtonTapped() {
Timber.v("## VOIP headSetButtonTapped")
val call = currentCall?.mxCall ?: return
@ -163,19 +141,11 @@ class WebRtcPeerConnectionManager @Inject constructor(
}
}
private suspend fun getTurnServer(): TurnServerResponse? {
return tryOrNull {
awaitCallback {
currentSession?.callSignalingService()?.getTurnServer(it)
}
}
}
fun attachViewRenderers(localViewRenderer: SurfaceViewRenderer?, remoteViewRenderer: SurfaceViewRenderer, mode: String?) {
currentCall?.attachViewRenderers(localViewRenderer, remoteViewRenderer, mode)
}
private fun createPeerConnectionFactory() {
private fun createPeerConnectionFactoryIfNeeded() {
if (peerConnectionFactory != null) return
Timber.v("## VOIP createPeerConnectionFactory")
val eglBaseContext = rootEglBase?.eglBaseContext ?: return Unit.also {
@ -202,12 +172,9 @@ class WebRtcPeerConnectionManager @Inject constructor(
.setVideoEncoderFactory(defaultVideoEncoderFactory)
.setVideoDecoderFactory(defaultVideoDecoderFactory)
.createPeerConnectionFactory()
// attachViewRenderersInternal()
}
fun acceptIncomingCall() {
Timber.v("## VOIP acceptIncomingCall from state ${currentCall?.mxCall?.state}")
currentCall?.acceptIncomingCall()
}
@ -215,11 +182,12 @@ class WebRtcPeerConnectionManager @Inject constructor(
currentCall?.detachRenderers(renderers)
}
fun close() {
Timber.v("## VOIP WebRtcPeerConnectionManager close() >")
private fun onCallEnded(call: WebRtcCall) {
Timber.v("## VOIP WebRtcPeerConnectionManager onCall ended: ${call.mxCall.callId}")
CallService.onNoActiveCall(context)
callAudioManager.stop()
currentCall = null
callsByCallId.remove(call.mxCall.callId)
// This must be done in this thread
executor.execute {
if (currentCall == null) {
@ -231,68 +199,28 @@ class WebRtcPeerConnectionManager @Inject constructor(
}
}
companion object {
private const val STREAM_ID = "ARDAMS"
private const val AUDIO_TRACK_ID = "ARDAMSa0"
private const val VIDEO_TRACK_ID = "ARDAMSv0"
private val DEFAULT_AUDIO_CONSTRAINTS = MediaConstraints().apply {
// add all existing audio filters to avoid having echos
// mandatory.add(MediaConstraints.KeyValuePair("googEchoCancellation", "true"))
// mandatory.add(MediaConstraints.KeyValuePair("googEchoCancellation2", "true"))
// mandatory.add(MediaConstraints.KeyValuePair("googDAEchoCancellation", "true"))
//
// mandatory.add(MediaConstraints.KeyValuePair("googTypingNoiseDetection", "true"))
//
// mandatory.add(MediaConstraints.KeyValuePair("googAutoGainControl", "true"))
// mandatory.add(MediaConstraints.KeyValuePair("googAutoGainControl2", "true"))
//
// mandatory.add(MediaConstraints.KeyValuePair("googNoiseSuppression", "true"))
// mandatory.add(MediaConstraints.KeyValuePair("googNoiseSuppression2", "true"))
//
// mandatory.add(MediaConstraints.KeyValuePair("googAudioMirroring", "false"))
// mandatory.add(MediaConstraints.KeyValuePair("googHighpassFilter", "true"))
}
}
fun startOutgoingCall(signalingRoomId: String, otherUserId: String, isVideoCall: Boolean) {
Timber.v("## VOIP startOutgoingCall in room $signalingRoomId to $otherUserId isVideo $isVideoCall")
executor.execute {
if (peerConnectionFactory == null) {
createPeerConnectionFactory()
}
createPeerConnectionFactoryIfNeeded()
}
val createdCall = currentSession?.callSignalingService()?.createOutgoingCall(signalingRoomId, otherUserId, isVideoCall) ?: return
val webRtcCall = WebRtcCall(
mxCall = createdCall,
callAudioManager = callAudioManager,
rootEglBase = rootEglBase,
context = context,
executor = executor,
peerConnectionFactoryProvider = Provider {
createPeerConnectionFactory()
peerConnectionFactory
},
session = currentSession!!
)
val mxCall = currentSession?.callSignalingService()?.createOutgoingCall(signalingRoomId, otherUserId, isVideoCall) ?: return
createWebRtcCall(mxCall)
callAudioManager.startForCall(mxCall)
callAudioManager.startForCall(createdCall)
currentCall = webRtcCall
val name = currentSession?.getUser(createdCall.opponentUserId)?.getBestName()
?: createdCall.opponentUserId
val name = currentSession?.getUser(mxCall.opponentUserId)?.getBestName()
?: mxCall.opponentUserId
CallService.onOutgoingCallRinging(
context = context.applicationContext,
isVideo = createdCall.isVideoCall,
isVideo = mxCall.isVideoCall,
roomName = name,
roomId = createdCall.roomId,
roomId = mxCall.roomId,
matrixId = currentSession?.myUserId ?: "",
callId = createdCall.callId)
callId = mxCall.callId)
// start the activity now
context.applicationContext.startActivity(VectorCallActivity.newIntent(context, createdCall))
context.startActivity(VectorCallActivity.newIntent(context, mxCall))
}
override fun onCallIceCandidateReceived(mxCall: MxCall, iceCandidatesContent: CallCandidatesContent) {
@ -311,19 +239,9 @@ class WebRtcPeerConnectionManager @Inject constructor(
// Just ignore, maybe we could answer from other session?
return
}
val webRtcCall = WebRtcCall(
mxCall = mxCall,
callAudioManager = callAudioManager,
rootEglBase = rootEglBase,
context = context,
executor = executor,
peerConnectionFactoryProvider = {
createPeerConnectionFactory()
peerConnectionFactory
},
session = currentSession!!
)
currentCall = webRtcCall
createWebRtcCall(mxCall).apply {
offerSdp = callInviteContent.offer
}
callAudioManager.startForCall(mxCall)
// Start background service with notification
val name = currentSession?.getUser(mxCall.opponentUserId)?.getBestName()
@ -336,8 +254,6 @@ class WebRtcPeerConnectionManager @Inject constructor(
matrixId = currentSession?.myUserId ?: "",
callId = mxCall.callId
)
webRtcCall.offerSdp = callInviteContent.offer
// If this is received while in background, the app will not sync,
// and thus won't be able to received events. For example if the call is
// accepted on an other session this device will continue ringing
@ -351,21 +267,23 @@ class WebRtcPeerConnectionManager @Inject constructor(
}
}
private suspend fun createAnswer(call: WebRtcCall): SessionDescription? {
Timber.w("## VOIP createAnswer")
val peerConnection = call.peerConnection ?: return null
val constraints = MediaConstraints().apply {
mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true"))
mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveVideo", if (call.mxCall.isVideoCall) "true" else "false"))
}
return try {
val localDescription = peerConnection.awaitCreateAnswer(constraints) ?: return null
peerConnection.awaitSetLocalDescription(localDescription)
localDescription
} catch (failure: Throwable) {
Timber.v("Fail to create answer")
null
}
private fun createWebRtcCall(mxCall: MxCall): WebRtcCall {
val webRtcCall = WebRtcCall(
mxCall = mxCall,
callAudioManager = callAudioManager,
rootEglBase = rootEglBase,
context = context,
dispatcher = dispatcher,
peerConnectionFactoryProvider = {
createPeerConnectionFactoryIfNeeded()
peerConnectionFactory
},
sessionProvider = { currentSession },
onCallEnded = this::onCallEnded
)
currentCall = webRtcCall
callsByCallId[mxCall.callId] = webRtcCall
return webRtcCall
}
fun muteCall(muted: Boolean) {
@ -397,11 +315,7 @@ class WebRtcPeerConnectionManager @Inject constructor(
}
fun endCall(originatedByMe: Boolean = true) {
// Update service state
CallService.onNoActiveCall(context)
// close tracks ASAP
currentCall?.endCall(originatedByMe)
close()
}
fun onWiredDeviceEvent(event: WiredHeadsetStateReceiver.HeadsetPlugEvent) {
@ -478,6 +392,7 @@ class WebRtcPeerConnectionManager @Inject constructor(
override fun onCallManagedByOtherSession(callId: String) {
Timber.v("## VOIP onCallManagedByOtherSession: $callId")
currentCall = null
callsByCallId.remove(callId)
CallService.onNoActiveCall(context)
// did we start background sync? so we should stop it

6
vector/src/main/java/im/vector/app/features/home/HomeDetailFragment.kt
View File

@ -35,7 +35,7 @@ import im.vector.app.core.ui.views.ActiveCallViewHolder
import im.vector.app.core.ui.views.KeysBackupBanner
import im.vector.app.features.call.SharedActiveCallViewModel
import im.vector.app.features.call.VectorCallActivity
import im.vector.app.features.call.webrtc.WebRtcPeerConnectionManager
import im.vector.app.features.call.webrtc.WebRtcCallManager
import im.vector.app.features.home.room.list.RoomListFragment
import im.vector.app.features.home.room.list.RoomListParams
import im.vector.app.features.popup.PopupAlertManager
@ -62,7 +62,7 @@ class HomeDetailFragment @Inject constructor(
private val serverBackupStatusViewModelFactory: ServerBackupStatusViewModel.Factory,
private val avatarRenderer: AvatarRenderer,
private val alertManager: PopupAlertManager,
private val webRtcPeerConnectionManager: WebRtcPeerConnectionManager,
private val callManager: WebRtcCallManager,
private val vectorPreferences: VectorPreferences
) : VectorBaseFragment(), KeysBackupBanner.Delegate, ActiveCallView.Callback, ServerBackupStatusViewModel.Factory {
@ -120,7 +120,7 @@ class HomeDetailFragment @Inject constructor(
sharedCallActionViewModel
.activeCall
.observe(viewLifecycleOwner, Observer {
activeCallViewHolder.updateCall(it, webRtcPeerConnectionManager)
activeCallViewHolder.updateCall(it, callManager)
invalidateOptionsMenu()
})
}

8
vector/src/main/java/im/vector/app/features/home/room/detail/RoomDetailFragment.kt
View File

@ -116,7 +116,7 @@ import im.vector.app.features.attachments.preview.AttachmentsPreviewArgs
import im.vector.app.features.attachments.toGroupedContentAttachmentData
import im.vector.app.features.call.SharedActiveCallViewModel
import im.vector.app.features.call.VectorCallActivity
import im.vector.app.features.call.webrtc.WebRtcPeerConnectionManager
import im.vector.app.features.call.webrtc.WebRtcCallManager
import im.vector.app.features.call.conference.JitsiCallViewModel
import im.vector.app.features.command.Command
import im.vector.app.features.crypto.keysbackup.restore.KeysBackupRestoreActivity
@ -218,7 +218,7 @@ class RoomDetailFragment @Inject constructor(
private val vectorPreferences: VectorPreferences,
private val colorProvider: ColorProvider,
private val notificationUtils: NotificationUtils,
private val webRtcPeerConnectionManager: WebRtcPeerConnectionManager,
private val callManager: WebRtcCallManager,
private val matrixItemColorProvider: MatrixItemColorProvider,
private val imageContentRenderer: ImageContentRenderer,
private val roomDetailPendingActionStore: RoomDetailPendingActionStore
@ -315,7 +315,7 @@ class RoomDetailFragment @Inject constructor(
sharedCallActionViewModel
.activeCall
.observe(viewLifecycleOwner, Observer {
activeCallViewHolder.updateCall(it, webRtcPeerConnectionManager)
activeCallViewHolder.updateCall(it, callManager)
invalidateOptionsMenu()
})
@ -514,7 +514,7 @@ class RoomDetailFragment @Inject constructor(
}
override fun onDestroy() {
activeCallViewHolder.unBind(webRtcPeerConnectionManager)
activeCallViewHolder.unBind(callManager)
roomDetailViewModel.handle(RoomDetailAction.ExitTrackingUnreadMessagesState)
super.onDestroy()
}

10
vector/src/main/java/im/vector/app/features/home/room/detail/RoomDetailViewModel.kt
View File

@ -32,7 +32,7 @@ import im.vector.app.core.extensions.exhaustive
import im.vector.app.core.platform.VectorViewModel
import im.vector.app.core.resources.StringProvider
import im.vector.app.core.utils.subscribeLogError
import im.vector.app.features.call.webrtc.WebRtcPeerConnectionManager
import im.vector.app.features.call.webrtc.WebRtcCallManager
import im.vector.app.features.command.CommandParser
import im.vector.app.features.command.ParsedCommand
import im.vector.app.features.crypto.verification.SupportedVerificationMethodsProvider
@ -114,7 +114,7 @@ class RoomDetailViewModel @AssistedInject constructor(
private val stickerPickerActionHandler: StickerPickerActionHandler,
private val roomSummaryHolder: RoomSummaryHolder,
private val typingHelper: TypingHelper,
private val webRtcPeerConnectionManager: WebRtcPeerConnectionManager,
private val callManager: WebRtcCallManager,
timelineSettingsFactory: TimelineSettingsFactory
) : VectorViewModel<RoomDetailViewState, RoomDetailAction, RoomDetailViewEvents>(initialState), Timeline.Listener {
@ -306,12 +306,12 @@ class RoomDetailViewModel @AssistedInject constructor(
private fun handleStartCall(action: RoomDetailAction.StartCall) {
room.roomSummary()?.otherMemberIds?.firstOrNull()?.let {
webRtcPeerConnectionManager.startOutgoingCall(room.roomId, it, action.isVideo)
callManager.startOutgoingCall(room.roomId, it, action.isVideo)
}
}
private fun handleEndCall() {
webRtcPeerConnectionManager.endCall()
callManager.endCall()
}
private fun handleSelectStickerAttachment() {
@ -566,7 +566,7 @@ class RoomDetailViewModel @AssistedInject constructor(
R.id.open_matrix_apps -> true
R.id.voice_call,
R.id.video_call -> true // always show for discoverability
R.id.hangup_call -> webRtcPeerConnectionManager.currentCall != null
R.id.hangup_call -> callManager.currentCall != null
R.id.search -> true
else -> false
}