From 743ace7e606a3556d60c2ecfb5e9b821d7d48917 Mon Sep 17 00:00:00 2001 From: onurays Date: Mon, 25 May 2020 23:28:30 +0300 Subject: [PATCH] Move voip responsibilities from views to WebRtcPeerConnectionManager. --- .../android/api/session/call/CallService.kt | 5 + .../session/call/DefaultCallService.kt | 10 ++ .../riotx/features/call/VectorCallActivity.kt | 44 +++-- .../features/call/VectorCallViewModel.kt | 33 +--- .../call/WebRtcPeerConnectionManager.kt | 150 +++++++++++------- .../features/call/telecom/CallConnection.kt | 12 +- 6 files changed, 139 insertions(+), 115 deletions(-) diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallService.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallService.kt index bb5a81907e..554c00ab05 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallService.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallService.kt @@ -46,6 +46,11 @@ interface CallService { */ fun sendLocalIceCandidateRemovals(callId: String, roomId: String, candidates: List) + /** + * Send a hangup event + */ + fun sendHangup(callId: String, roomId: String) + fun addCallListener(listener: CallsListener) fun removeCallListener(listener: CallsListener) diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt index fba50a0c37..abc17cf89c 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt @@ -127,6 +127,16 @@ internal class DefaultCallService @Inject constructor( override fun sendLocalIceCandidateRemovals(callId: String, roomId: String, candidates: List) { } + override fun sendHangup(callId: String, roomId: String) { + val eventContent = CallHangupContent( + callId = callId, + version = 0 + ) + createEventAndLocalEcho(type = EventType.CALL_HANGUP, roomId = roomId, content = eventContent.toContent()).let { event -> + roomEventSender.sendEvent(event) + } + } + override fun addCallListener(listener: CallsListener) { if (!callListeners.contains(listener)) callListeners.add(listener) } diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt index fef5aa0647..3f6b38957b 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt @@ -38,17 +38,14 @@ import im.vector.riotx.core.utils.checkPermissions import io.reactivex.android.schedulers.AndroidSchedulers import io.reactivex.subjects.PublishSubject import kotlinx.android.parcel.Parcelize -import org.webrtc.Camera1Enumerator -import org.webrtc.Camera2Enumerator +import kotlinx.android.synthetic.main.activity_call.* import org.webrtc.EglBase import org.webrtc.IceCandidate import org.webrtc.MediaStream -import org.webrtc.PeerConnection import org.webrtc.RendererCommon import org.webrtc.SessionDescription import org.webrtc.SurfaceViewRenderer import org.webrtc.VideoTrack -import java.util.concurrent.TimeUnit import javax.inject.Inject @Parcelize @@ -69,6 +66,7 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis } private val callViewModel: VectorCallViewModel by viewModel() + private lateinit var callArgs: CallArgs @Inject lateinit var peerConnectionManager: WebRtcPeerConnectionManager @@ -114,10 +112,19 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) + + if (intent.hasExtra(MvRx.KEY_ARG)) { + callArgs = intent.getParcelableExtra(MvRx.KEY_ARG)!! + } else { + finish() + } + rootEglBase = EglUtils.rootEglBase ?: return Unit.also { finish() } + iv_end_call.setOnClickListener { callViewModel.handle(VectorCallViewActions.EndCall) } + callViewModel.viewEvents .observe() .observeOn(AndroidSchedulers.mainThread()) @@ -161,8 +168,9 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis // setSwappedFeeds(true /* isSwappedFeeds */); if (isFirstCreation()) { - peerConnectionManager.createPeerConnectionFactory() + //peerConnectionManager.createPeerConnectionFactory() + /* val cameraIterator = if (Camera2Enumerator.isSupported(this)) Camera2Enumerator(this) else Camera1Enumerator(false) val frontCamera = cameraIterator.deviceNames ?.firstOrNull { cameraIterator.isFrontFacing(it) } @@ -170,19 +178,12 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis ?: return true val videoCapturer = cameraIterator.createCapturer(frontCamera, null) - val iceServers = ArrayList().apply { - listOf("turn:turn.matrix.org:3478?transport=udp", "turn:turn.matrix.org:3478?transport=tcp", "turns:turn.matrix.org:443?transport=tcp").forEach { - add( - PeerConnection.IceServer.builder(it) - .setUsername("xxxxx") - .setPassword("xxxxx") - .createIceServer() - ) - } - } + peerConnectionManager.createPeerConnection(videoCapturer, iceServers) - peerConnectionManager.startCall() + */ + + //peerConnectionManager.startCall() } // PeerConnectionFactory.initialize(PeerConnectionFactory // .InitializationOptions.builder(applicationContext) @@ -322,15 +323,6 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis // Timber.v("## VOIP onCreateFailure $p0") // } // }, constraints) - iceCandidateSource - .buffer(400, TimeUnit.MILLISECONDS) - .subscribe { - // omit empty :/ - if (it.isNotEmpty()) { - callViewModel.handle(VectorCallViewActions.AddLocalIceCandidate(it)) - } - } - .disposeOnDestroy() peerConnectionManager.attachViewRenderers(pipRenderer, fullscreenRenderer) return false @@ -433,6 +425,6 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis } override fun sendOffer(sessionDescription: SessionDescription) { - callViewModel.handle(VectorCallViewActions.SendOffer(sessionDescription)) + } } diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt index 4613c6200b..d6270ad936 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt @@ -16,7 +16,6 @@ package im.vector.riotx.features.call -import androidx.lifecycle.viewModelScope import com.airbnb.mvrx.MvRxState import com.airbnb.mvrx.MvRxViewModelFactory import com.airbnb.mvrx.ViewModelContext @@ -27,16 +26,10 @@ import im.vector.matrix.android.api.session.call.CallsListener import im.vector.matrix.android.api.session.room.model.call.CallAnswerContent import im.vector.matrix.android.api.session.room.model.call.CallHangupContent import im.vector.matrix.android.api.session.room.model.call.CallInviteContent -import im.vector.matrix.android.internal.util.awaitCallback import im.vector.riotx.core.extensions.exhaustive import im.vector.riotx.core.platform.VectorViewEvents import im.vector.riotx.core.platform.VectorViewModel import im.vector.riotx.core.platform.VectorViewModelAction -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.launch -import org.webrtc.IceCandidate -import org.webrtc.SessionDescription -import java.util.UUID data class VectorCallViewState( val callId: String? = null, @@ -45,8 +38,7 @@ data class VectorCallViewState( sealed class VectorCallViewActions : VectorViewModelAction { - data class SendOffer(val sdp: SessionDescription) : VectorCallViewActions() - data class AddLocalIceCandidate(val iceCandidates: List) : VectorCallViewActions() + object EndCall : VectorCallViewActions() } sealed class VectorCallViewEvents : VectorViewEvents { @@ -57,7 +49,8 @@ sealed class VectorCallViewEvents : VectorViewEvents { class VectorCallViewModel @AssistedInject constructor( @Assisted initialState: VectorCallViewState, - val session: Session + val session: Session, + val webRtcPeerConnectionManager: WebRtcPeerConnectionManager ) : VectorViewModel(initialState) { private val callServiceListener: CallsListener = object : CallsListener { @@ -90,25 +83,9 @@ class VectorCallViewModel @AssistedInject constructor( super.onCleared() } - override fun handle(action: VectorCallViewActions) = withState { state -> + override fun handle(action: VectorCallViewActions) = withState { when (action) { - is VectorCallViewActions.SendOffer -> { - viewModelScope.launch(Dispatchers.IO) { - awaitCallback { - val callId = state.callId ?: UUID.randomUUID().toString().also { - setState { - copy(callId = it) - } - } - session.callService().sendOfferSdp(callId, state.roomId, action.sdp, it) - } - } - } - is VectorCallViewActions.AddLocalIceCandidate -> { - viewModelScope.launch { - session.callService().sendLocalIceCandidates(state.callId ?: "", state.roomId, action.iceCandidates) - } - } + VectorCallViewActions.EndCall -> webRtcPeerConnectionManager.endCall() }.exhaustive } diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index 3ff40afdbb..b466ea9f11 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -22,6 +22,7 @@ import android.content.Intent import android.content.ServiceConnection import android.os.IBinder import androidx.core.content.ContextCompat +import im.vector.matrix.android.api.MatrixCallback import im.vector.matrix.android.api.session.call.CallsListener import im.vector.matrix.android.api.session.call.EglUtils import im.vector.matrix.android.api.session.room.model.call.CallAnswerContent @@ -29,6 +30,8 @@ import im.vector.matrix.android.api.session.room.model.call.CallHangupContent import im.vector.matrix.android.api.session.room.model.call.CallInviteContent import im.vector.riotx.core.di.ActiveSessionHolder import im.vector.riotx.features.call.service.CallHeadsUpService +import io.reactivex.disposables.Disposable +import io.reactivex.subjects.PublishSubject import org.webrtc.AudioSource import org.webrtc.AudioTrack import org.webrtc.DefaultVideoDecoderFactory @@ -47,7 +50,9 @@ import org.webrtc.VideoSource import org.webrtc.VideoTrack import timber.log.Timber import java.lang.ref.WeakReference +import java.util.UUID import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit import javax.inject.Inject import javax.inject.Singleton @@ -59,7 +64,7 @@ import javax.inject.Singleton class WebRtcPeerConnectionManager @Inject constructor( private val context: Context, private val sessionHolder: ActiveSessionHolder - ) : CallsListener { +) : CallsListener { interface Listener { fun addLocalIceCandidate(candidates: IceCandidate) @@ -98,8 +103,15 @@ class WebRtcPeerConnectionManager @Inject constructor( var localSurfaceRenderer: WeakReference? = null var remoteSurfaceRenderer: WeakReference? = null + private val iceCandidateSource: PublishSubject = PublishSubject.create() + private var iceCandidateDisposable: Disposable? = null + var callHeadsUpService: CallHeadsUpService? = null + private var callId: String? = null + private var signalingRoomId: String? = null + private var participantUserId: String? = null + private val serviceConnection = object : ServiceConnection { override fun onServiceDisconnected(name: ComponentName?) { } @@ -109,7 +121,7 @@ class WebRtcPeerConnectionManager @Inject constructor( } } - fun createPeerConnectionFactory() { + private fun createPeerConnectionFactory() { executor.execute { if (peerConnectionFactory == null) { Timber.v("## VOIP createPeerConnectionFactory") @@ -142,7 +154,56 @@ class WebRtcPeerConnectionManager @Inject constructor( } } - fun createPeerConnection(videoCapturer: VideoCapturer, iceServers: List) { + private fun createPeerConnection() { + val iceServers = ArrayList().apply { + listOf("turn:turn.matrix.org:3478?transport=udp", "turn:turn.matrix.org:3478?transport=tcp", "turns:turn.matrix.org:443?transport=tcp").forEach { + add( + PeerConnection.IceServer.builder(it) + .setUsername("xxxxx") + .setPassword("xxxxx") + .createIceServer() + ) + } + } + Timber.v("## VOIP creating peer connection... ") + peerConnection = peerConnectionFactory?.createPeerConnection( + iceServers, + object : PeerConnectionObserverAdapter() { + override fun onIceCandidate(p0: IceCandidate?) { + Timber.v("## VOIP onIceCandidate local $p0") + p0?.let { iceCandidateSource.onNext(it) } + } + + override fun onAddStream(mediaStream: MediaStream?) { + Timber.v("## VOIP onAddStream remote $mediaStream") + mediaStream?.videoTracks?.firstOrNull()?.let { + listener?.addRemoteVideoTrack(it) + remoteVideoTrack = it + } + } + + override fun onRemoveStream(mediaStream: MediaStream?) { + mediaStream?.let { + listener?.removeRemoteVideoStream(it) + } + remoteSurfaceRenderer?.get()?.let { + remoteVideoTrack?.removeSink(it) + } + remoteVideoTrack = null + } + + override fun onIceConnectionChange(p0: PeerConnection.IceConnectionState?) { + Timber.v("## VOIP onIceConnectionChange $p0") + if (p0 == PeerConnection.IceConnectionState.DISCONNECTED) { + listener?.onDisconnect() + } + } + } + ) + } + + // TODO REMOVE THIS FUNCTION + private fun createPeerConnection(videoCapturer: VideoCapturer) { executor.execute { Timber.v("## VOIP PeerConnectionFactory.createPeerConnection $peerConnectionFactory...") // Following instruction here: https://stackoverflow.com/questions/55085726/webrtc-create-peerconnectionfactory-object @@ -183,55 +244,7 @@ class WebRtcPeerConnectionManager @Inject constructor( // } // .disposeOnDestroy() - Timber.v("## VOIP creating peer connection... ") - peerConnection = peerConnectionFactory?.createPeerConnection( - iceServers, - object : PeerConnectionObserverAdapter() { - override fun onIceCandidate(p0: IceCandidate?) { - Timber.v("## VOIP onIceCandidate local $p0") - p0?.let { - // iceCandidateSource.onNext(it) - listener?.addLocalIceCandidate(it) - } - } - override fun onAddStream(mediaStream: MediaStream?) { - Timber.v("## VOIP onAddStream remote $mediaStream") - mediaStream?.videoTracks?.firstOrNull()?.let { - listener?.addRemoteVideoTrack(it) - remoteVideoTrack = it -// remoteSurfaceRenderer?.get()?.let { surface -> -// it.setEnabled(true) -// it.addSink(surface) -// } - } -// runOnUiThread { -// mediaStream?.videoTracks?.firstOrNull()?.let { videoTrack -> -// remoteVideoTrack = videoTrack -// remoteVideoTrack?.setEnabled(true) -// remoteVideoTrack?.addSink(fullscreenRenderer) -// } -// } - } - - override fun onRemoveStream(mediaStream: MediaStream?) { - mediaStream?.let { - listener?.removeRemoteVideoStream(it) - } - remoteSurfaceRenderer?.get()?.let { - remoteVideoTrack?.removeSink(it) - } - remoteVideoTrack = null - } - - override fun onIceConnectionChange(p0: PeerConnection.IceConnectionState?) { - Timber.v("## VOIP onIceConnectionChange $p0") - if (p0 == PeerConnection.IceConnectionState.DISCONNECTED) { - listener?.onDisconnect() - } - } - } - ) localMediaStream = peerConnectionFactory?.createLocalMediaStream("ARDAMS") // magic value? localMediaStream?.addTrack(localVideoTrack) @@ -253,7 +266,22 @@ class WebRtcPeerConnectionManager @Inject constructor( } } - fun startCall() { + private fun startCall() { + createPeerConnectionFactory() + createPeerConnection() + + iceCandidateDisposable = iceCandidateSource + .buffer(400, TimeUnit.MILLISECONDS) + .subscribe { + // omit empty :/ + if (it.isNotEmpty()) { + sessionHolder + .getActiveSession() + .callService() + .sendLocalIceCandidates(callId ?: "", signalingRoomId ?: "", it) + } + } + executor.execute { val constraints = MediaConstraints() constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")) @@ -273,8 +301,8 @@ class WebRtcPeerConnectionManager @Inject constructor( Timber.v("## VOIP onCreateSuccess $sessionDescription") peerConnection?.setLocalDescription(object : SdpObserverAdapter() { override fun onSetSuccess() { - listener?.sendOffer(sessionDescription) - // callViewModel.handle(VectorCallViewActions.SendOffer(sessionDescription)) + callId = UUID.randomUUID().toString() + sessionHolder.getActiveSession().callService().sendOfferSdp(callId!!, signalingRoomId!!, sessionDescription, object : MatrixCallback {}) } }, sessionDescription) } @@ -319,6 +347,7 @@ class WebRtcPeerConnectionManager @Inject constructor( peerConnectionFactory?.stopAecDump() peerConnectionFactory = null } + iceCandidateDisposable?.dispose() context.stopService(Intent(context, CallHeadsUpService::class.java)) } @@ -346,12 +375,18 @@ class WebRtcPeerConnectionManager @Inject constructor( } fun startOutgoingCall(context: Context, signalingRoomId: String, participantUserId: String, isVideoCall: Boolean) { + this.signalingRoomId = signalingRoomId + this.participantUserId = participantUserId startHeadsUpService(signalingRoomId, sessionHolder.getActiveSession().myUserId, false, isVideoCall) context.startActivity(VectorCallActivity.newIntent(context, signalingRoomId, participantUserId, false, isVideoCall)) + + startCall() } override fun onCallInviteReceived(signalingRoomId: String, participantUserId: String, callInviteContent: CallInviteContent) { startHeadsUpService(signalingRoomId, participantUserId, true, callInviteContent.isVideo()) + + startCall() } private fun startHeadsUpService(roomId: String, participantUserId: String, isIncomingCall: Boolean, isVideoCall: Boolean) { @@ -361,6 +396,13 @@ class WebRtcPeerConnectionManager @Inject constructor( context.bindService(Intent(context, CallHeadsUpService::class.java), serviceConnection, 0) } + fun endCall() { + if (callId != null && signalingRoomId != null) { + sessionHolder.getActiveSession().callService().sendHangup(callId!!, signalingRoomId!!) + close() + } + } + override fun onCallAnswerReceived(callAnswerContent: CallAnswerContent) { } diff --git a/vector/src/main/java/im/vector/riotx/features/call/telecom/CallConnection.kt b/vector/src/main/java/im/vector/riotx/features/call/telecom/CallConnection.kt index 27bdf24570..6f0c121b3d 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/telecom/CallConnection.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/telecom/CallConnection.kt @@ -21,14 +21,10 @@ import android.os.Build import android.telecom.Connection import android.telecom.DisconnectCause import androidx.annotation.RequiresApi -import im.vector.riotx.features.call.VectorCallViewActions import im.vector.riotx.features.call.VectorCallViewModel import im.vector.riotx.features.call.WebRtcPeerConnectionManager -import org.webrtc.Camera1Enumerator -import org.webrtc.Camera2Enumerator import org.webrtc.IceCandidate import org.webrtc.MediaStream -import org.webrtc.PeerConnection import org.webrtc.SessionDescription import org.webrtc.VideoTrack import timber.log.Timber @@ -91,7 +87,8 @@ import javax.inject.Inject } private fun startCall() { - peerConnectionManager.createPeerConnectionFactory() + /* + //peerConnectionManager.createPeerConnectionFactory() peerConnectionManager.listener = this val cameraIterator = if (Camera2Enumerator.isSupported(context)) Camera2Enumerator(context) else Camera1Enumerator(false) @@ -113,7 +110,8 @@ import javax.inject.Inject } peerConnectionManager.createPeerConnection(videoCapturer, iceServers) - peerConnectionManager.startCall() + //peerConnectionManager.startCall() + */ } override fun addLocalIceCandidate(candidates: IceCandidate) { @@ -129,6 +127,6 @@ import javax.inject.Inject } override fun sendOffer(sessionDescription: SessionDescription) { - callViewModel.handle(VectorCallViewActions.SendOffer(sessionDescription)) + } }