diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt index c21251c9ed..e2a9888e82 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt @@ -31,6 +31,7 @@ import android.view.WindowManager import butterknife.BindView import com.airbnb.mvrx.MvRx import com.airbnb.mvrx.viewModel +import im.vector.matrix.android.api.extensions.tryThis import im.vector.matrix.android.api.session.call.EglUtils import im.vector.matrix.android.api.session.call.MxCallDetail import im.vector.riotx.R @@ -41,16 +42,11 @@ import im.vector.riotx.core.utils.allGranted import im.vector.riotx.core.utils.checkPermissions import im.vector.riotx.features.call.service.CallHeadsUpService import io.reactivex.android.schedulers.AndroidSchedulers -import io.reactivex.subjects.PublishSubject import kotlinx.android.parcel.Parcelize import kotlinx.android.synthetic.main.activity_call.* import org.webrtc.EglBase -import org.webrtc.IceCandidate -import org.webrtc.MediaStream import org.webrtc.RendererCommon -import org.webrtc.SessionDescription import org.webrtc.SurfaceViewRenderer -import org.webrtc.VideoTrack import javax.inject.Inject @Parcelize @@ -61,7 +57,7 @@ data class CallArgs( val isVideoCall: Boolean ) : Parcelable -class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Listener { +class VectorCallActivity : VectorBaseActivity() { override fun getLayoutRes() = R.layout.activity_call @@ -85,14 +81,6 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis private var rootEglBase: EglBase? = null -// private var peerConnectionFactory: PeerConnectionFactory? = null - - // private var peerConnection: PeerConnection? = null - -// private var remoteVideoTrack: VideoTrack? = null - - private val iceCandidateSource: PublishSubject = PublishSubject.create() - var callHeadsUpService: CallHeadsUpService? = null private val serviceConnection = object : ServiceConnection { override fun onServiceDisconnected(name: ComponentName?) { @@ -129,6 +117,7 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) + tryThis { unbindService(serviceConnection) } bindService(Intent(this, CallHeadsUpService::class.java), serviceConnection, 0) if (intent.hasExtra(MvRx.KEY_ARG)) { @@ -153,15 +142,10 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis handleViewEvents(it) } .disposeOnDestroy() -// -// if (isFirstCreation()) { -// -// } if (checkPermissions(PERMISSIONS_FOR_VIDEO_IP_CALL, this, CAPTURE_PERMISSION_REQUEST_CODE, R.string.permissions_rationale_msg_camera_and_audio)) { start() } - peerConnectionManager.listener = this } override fun onRequestPermissionsResult(requestCode: Int, permissions: Array, grantResults: IntArray) { @@ -185,165 +169,7 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis pipRenderer.setZOrderMediaOverlay(true) pipRenderer.setEnableHardwareScaler(true /* enabled */) fullscreenRenderer.setEnableHardwareScaler(true /* enabled */) - // Start with local feed in fullscreen and swap it to the pip when the call is connected. - // setSwappedFeeds(true /* isSwappedFeeds */); - if (isFirstCreation()) { - //peerConnectionManager.createPeerConnectionFactory() - - /* - val cameraIterator = if (Camera2Enumerator.isSupported(this)) Camera2Enumerator(this) else Camera1Enumerator(false) - val frontCamera = cameraIterator.deviceNames - ?.firstOrNull { cameraIterator.isFrontFacing(it) } - ?: cameraIterator.deviceNames?.first() - ?: return true - val videoCapturer = cameraIterator.createCapturer(frontCamera, null) - - - - peerConnectionManager.createPeerConnection(videoCapturer, iceServers) - */ - - //peerConnectionManager.startCall() - } -// PeerConnectionFactory.initialize(PeerConnectionFactory -// .InitializationOptions.builder(applicationContext) -// .createInitializationOptions() -// ) - -// val options = PeerConnectionFactory.Options() -// val defaultVideoEncoderFactory = DefaultVideoEncoderFactory( -// rootEglBase!!.eglBaseContext, /* enableIntelVp8Encoder */ -// true, /* enableH264HighProfile */ -// true) -// val defaultVideoDecoderFactory = DefaultVideoDecoderFactory(rootEglBase!!.eglBaseContext) -// -// peerConnectionFactory = PeerConnectionFactory.builder() -// .setOptions(options) -// .setVideoEncoderFactory(defaultVideoEncoderFactory) -// .setVideoDecoderFactory(defaultVideoDecoderFactory) -// .createPeerConnectionFactory() - -// val cameraIterator = if (Camera2Enumerator.isSupported(this)) Camera2Enumerator(this) else Camera1Enumerator(false) -// val frontCamera = cameraIterator.deviceNames -// ?.firstOrNull { cameraIterator.isFrontFacing(it) } -// ?: cameraIterator.deviceNames?.first() -// ?: return true -// val videoCapturer = cameraIterator.createCapturer(frontCamera, null) -// -// // Following instruction here: https://stackoverflow.com/questions/55085726/webrtc-create-peerconnectionfactory-object -// val surfaceTextureHelper = SurfaceTextureHelper.create("CaptureThread", rootEglBase!!.eglBaseContext) -// -// val videoSource = peerConnectionFactory?.createVideoSource(videoCapturer.isScreencast) -// videoCapturer.initialize(surfaceTextureHelper, this, videoSource!!.capturerObserver) -// videoCapturer.startCapture(1280, 720, 30) -// -// -// val localVideoTrack = peerConnectionFactory?.createVideoTrack("ARDAMSv0", videoSource) -// -// // create a local audio track -// val audioSource = peerConnectionFactory?.createAudioSource(DEFAULT_AUDIO_CONSTRAINTS) -// val audioTrack = peerConnectionFactory?.createAudioTrack("ARDAMSa0", audioSource) - - pipRenderer.setMirror(true) -// localVideoTrack?.addSink(pipRenderer) - - /* - { - "username": "1586847781:@valere35:matrix.org", - "password": "ZzbqbqfT9O2G3WpCpesdts2lyns=", - "ttl": 86400.0, - "uris": ["turn:turn.matrix.org:3478?transport=udp", "turn:turn.matrix.org:3478?transport=tcp", "turns:turn.matrix.org:443?transport=tcp"] - } - */ - -// val iceServers = ArrayList().apply { -// listOf("turn:turn.matrix.org:3478?transport=udp", "turn:turn.matrix.org:3478?transport=tcp", "turns:turn.matrix.org:443?transport=tcp").forEach { -// add( -// PeerConnection.IceServer.builder(it) -// .setUsername("1586847781:@valere35:matrix.org") -// .setPassword("ZzbqbqfT9O2G3WpCpesdts2lyns=") -// .createIceServer() -// ) -// } -// } -// -// val iceCandidateSource: PublishSubject = PublishSubject.create() -// -// iceCandidateSource -// .buffer(400, TimeUnit.MILLISECONDS) -// .subscribe { -// // omit empty :/ -// if (it.isNotEmpty()) { -// callViewModel.handle(VectorCallViewActions.AddLocalIceCandidate(it)) -// } -// } -// .disposeOnDestroy() -// -// peerConnection = peerConnectionFactory?.createPeerConnection( -// iceServers, -// object : PeerConnectionObserverAdapter() { -// override fun onIceCandidate(p0: IceCandidate?) { -// p0?.let { -// iceCandidateSource.onNext(it) -// } -// } -// -// override fun onAddStream(mediaStream: MediaStream?) { -// runOnUiThread { -// mediaStream?.videoTracks?.firstOrNull()?.let { videoTrack -> -// remoteVideoTrack = videoTrack -// remoteVideoTrack?.setEnabled(true) -// remoteVideoTrack?.addSink(fullscreenRenderer) -// } -// } -// } -// -// override fun onRemoveStream(mediaStream: MediaStream?) { -// remoteVideoTrack = null -// } -// -// override fun onIceConnectionChange(p0: PeerConnection.IceConnectionState?) { -// if (p0 == PeerConnection.IceConnectionState.DISCONNECTED) { -// // TODO prompt something? -// finish() -// } -// } -// } -// ) -// -// val localMediaStream = peerConnectionFactory?.createLocalMediaStream("ARDAMS") // magic value? -// localMediaStream?.addTrack(localVideoTrack) -// localMediaStream?.addTrack(audioTrack) -// -// val constraints = MediaConstraints() -// constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")) -// constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveVideo", "true")) -// -// peerConnection?.addStream(localMediaStream) -// -// peerConnection?.createOffer(object : SdpObserver { -// override fun onSetFailure(p0: String?) { -// Timber.v("## VOIP onSetFailure $p0") -// } -// -// override fun onSetSuccess() { -// Timber.v("## VOIP onSetSuccess") -// } -// -// override fun onCreateSuccess(sessionDescription: SessionDescription) { -// Timber.v("## VOIP onCreateSuccess $sessionDescription") -// peerConnection?.setLocalDescription(object : SdpObserverAdapter() { -// override fun onSetSuccess() { -// callViewModel.handle(VectorCallViewActions.SendOffer(sessionDescription)) -// } -// }, sessionDescription) -// } -// -// override fun onCreateFailure(p0: String?) { -// Timber.v("## VOIP onCreateFailure $p0") -// } -// }, constraints) peerConnectionManager.attachViewRenderers(pipRenderer, fullscreenRenderer) return false @@ -351,16 +177,13 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis override fun onDestroy() { peerConnectionManager.detachRenderers() - peerConnectionManager.listener = this + tryThis { unbindService(serviceConnection) } super.onDestroy() } private fun handleViewEvents(event: VectorCallViewEvents?) { when (event) { is VectorCallViewEvents.CallAnswered -> { - //val sdp = SessionDescription(SessionDescription.Type.ANSWER, event.content.answer.sdp) - //peerConnectionManager.answerReceived("", sdp) -// peerConnection?.setRemoteDescription(object : SdpObserverAdapter() {}, sdp) } is VectorCallViewEvents.CallHangup -> { finish() @@ -368,53 +191,10 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis } } -// @TargetApi(17) -// private fun getDisplayMetrics(): DisplayMetrics? { -// val displayMetrics = DisplayMetrics() -// val windowManager = application.getSystemService(Context.WINDOW_SERVICE) as WindowManager -// windowManager.defaultDisplay.getRealMetrics(displayMetrics) -// return displayMetrics -// } - -// @TargetApi(21) -// private fun startScreenCapture() { -// val mediaProjectionManager: MediaProjectionManager = application.getSystemService( -// Context.MEDIA_PROJECTION_SERVICE) as MediaProjectionManager -// startActivityForResult( -// mediaProjectionManager.createScreenCaptureIntent(), CAPTURE_PERMISSION_REQUEST_CODE) -// } -// -// override fun onActivityResult(requestCode: Int, resultCode: Int, data: Intent?) { -// if (requestCode != CAPTURE_PERMISSION_REQUEST_CODE) { -// super.onActivityResult(requestCode, resultCode, data) -// } -// // mediaProjectionPermissionResultCode = resultCode; -// // mediaProjectionPermissionResultData = data; -// // startCall(); -// } - companion object { private const val CAPTURE_PERMISSION_REQUEST_CODE = 1 -// private val DEFAULT_AUDIO_CONSTRAINTS = MediaConstraints().apply { -// // add all existing audio filters to avoid having echos -// mandatory.add(MediaConstraints.KeyValuePair("googEchoCancellation", "true")) -// mandatory.add(MediaConstraints.KeyValuePair("googEchoCancellation2", "true")) -// mandatory.add(MediaConstraints.KeyValuePair("googDAEchoCancellation", "true")) -// -// mandatory.add(MediaConstraints.KeyValuePair("googTypingNoiseDetection", "true")) -// -// mandatory.add(MediaConstraints.KeyValuePair("googAutoGainControl", "true")) -// mandatory.add(MediaConstraints.KeyValuePair("googAutoGainControl2", "true")) -// -// mandatory.add(MediaConstraints.KeyValuePair("googNoiseSuppression", "true")) -// mandatory.add(MediaConstraints.KeyValuePair("googNoiseSuppression2", "true")) -// -// mandatory.add(MediaConstraints.KeyValuePair("googAudioMirroring", "false")) -// mandatory.add(MediaConstraints.KeyValuePair("googHighpassFilter", "true")) -// } - fun newIntent(context: Context, mxCall: MxCallDetail): Intent { return Intent(context, VectorCallActivity::class.java).apply { flags = Intent.FLAG_ACTIVITY_NEW_TASK @@ -422,30 +202,4 @@ class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Lis } } } - - override fun addLocalIceCandidate(candidates: IceCandidate) { - iceCandidateSource.onNext(candidates) - } - - override fun addRemoteVideoTrack(videoTrack: VideoTrack) { - runOnUiThread { - videoTrack.setEnabled(true) - videoTrack.addSink(fullscreenRenderer) - } - } - - override fun addLocalVideoTrack(videoTrack: VideoTrack) { - runOnUiThread { - videoTrack.addSink(pipRenderer) - } - } - - override fun removeRemoteVideoStream(mediaStream: MediaStream) { - } - - override fun onDisconnect() { - } - - override fun sendOffer(sessionDescription: SessionDescription) { - } } diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index ec5f6f2048..4e70aef5b5 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -69,19 +69,8 @@ class WebRtcPeerConnectionManager @Inject constructor( private val sessionHolder: ActiveSessionHolder ) : CallsListener { - interface Listener { - fun addLocalIceCandidate(candidates: IceCandidate) - fun addRemoteVideoTrack(videoTrack: VideoTrack) - fun addLocalVideoTrack(videoTrack: VideoTrack) - fun removeRemoteVideoStream(mediaStream: MediaStream) - fun onDisconnect() - fun sendOffer(sessionDescription: SessionDescription) - } - var localMediaStream: MediaStream? = null - var listener: Listener? = null - // *Comments copied from webrtc demo app* // Executor thread is started once and is used for all // peer connection API calls to ensure new peer connection factory is @@ -102,7 +91,7 @@ class WebRtcPeerConnectionManager @Inject constructor( private var videoSource: VideoSource? = null private var audioSource: AudioSource? = null - private var audioTrack: AudioTrack? = null + private var localAudioTrack: AudioTrack? = null private var videoCapturer: VideoCapturer? = null @@ -171,60 +160,6 @@ class WebRtcPeerConnectionManager @Inject constructor( peerConnection = peerConnectionFactory?.createPeerConnection(iceServers, observer) } - // TODO REMOVE THIS FUNCTION - private fun createPeerConnection(videoCapturer: VideoCapturer) { - executor.execute { - Timber.v("## VOIP PeerConnectionFactory.createPeerConnection $peerConnectionFactory...") - // Following instruction here: https://stackoverflow.com/questions/55085726/webrtc-create-peerconnectionfactory-object - val surfaceTextureHelper = SurfaceTextureHelper.create("CaptureThread", rootEglBase!!.eglBaseContext) - - videoSource = peerConnectionFactory?.createVideoSource(videoCapturer.isScreencast) - Timber.v("## VOIP Local video source created") - videoCapturer.initialize(surfaceTextureHelper, context.applicationContext, videoSource!!.capturerObserver) - videoCapturer.startCapture(1280, 720, 30) - - localVideoTrack = peerConnectionFactory?.createVideoTrack("ARDAMSv0", videoSource)?.also { - Timber.v("## VOIP Local video track created") -// localSurfaceRenderer?.get()?.let { surface -> -// // it.addSink(surface) -// // } - } - - // create a local audio track - Timber.v("## VOIP create local audio track") - audioSource = peerConnectionFactory?.createAudioSource(DEFAULT_AUDIO_CONSTRAINTS) - audioTrack = peerConnectionFactory?.createAudioTrack(AUDIO_TRACK_ID, audioSource) - -// pipRenderer.setMirror(true) -// localVideoTrack?.addSink(pipRenderer) -// - -// val iceCandidateSource: PublishSubject = PublishSubject.create() -// -// iceCandidateSource -// .buffer(400, TimeUnit.MILLISECONDS) -// .subscribe { -// // omit empty :/ -// if (it.isNotEmpty()) { -// listener.addLocalIceCandidate() -// callViewModel.handle(VectorCallViewActions.AddLocalIceCandidate(it)) -// } -// } -// .disposeOnDestroy() - - localMediaStream = peerConnectionFactory?.createLocalMediaStream("ARDAMS") // magic value? - localMediaStream?.addTrack(localVideoTrack) - localMediaStream?.addTrack(audioTrack) - -// val constraints = MediaConstraints() -// constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")) -// constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveVideo", "true")) - - Timber.v("## VOIP add local stream to peer connection") - peerConnection?.addStream(localMediaStream) - } - } - private fun startCall() { createPeerConnectionFactory() createPeerConnection(object : PeerConnectionObserverAdapter() { @@ -244,15 +179,12 @@ class WebRtcPeerConnectionManager @Inject constructor( mediaStream.videoTracks?.firstOrNull()?.let { videoTrack -> remoteVideoTrack = videoTrack remoteVideoTrack?.setEnabled(true) - remoteVideoTrack?.addSink(remoteViewRenderer) + remoteViewRenderer?.let { remoteVideoTrack?.addSink(it) } } } } override fun onRemoveStream(mediaStream: MediaStream?) { - mediaStream?.let { - listener?.removeRemoteVideoStream(it) - } remoteSurfaceRenderer?.get()?.let { remoteVideoTrack?.removeSink(it) } @@ -262,7 +194,7 @@ class WebRtcPeerConnectionManager @Inject constructor( override fun onIceConnectionChange(p0: PeerConnection.IceConnectionState?) { Timber.v("## VOIP onIceConnectionChange $p0") if (p0 == PeerConnection.IceConnectionState.DISCONNECTED) { - listener?.onDisconnect() + endCall() } } }) @@ -313,8 +245,12 @@ class WebRtcPeerConnectionManager @Inject constructor( fun attachViewRenderers(localViewRenderer: SurfaceViewRenderer, remoteViewRenderer: SurfaceViewRenderer) { this.localViewRenderer = localViewRenderer this.remoteViewRenderer = remoteViewRenderer + this.localSurfaceRenderer = WeakReference(localViewRenderer) + this.remoteSurfaceRenderer = WeakReference(remoteViewRenderer) + audioSource = peerConnectionFactory?.createAudioSource(DEFAULT_AUDIO_CONSTRAINTS) - audioTrack = peerConnectionFactory?.createAudioTrack(AUDIO_TRACK_ID, audioSource) + localAudioTrack = peerConnectionFactory?.createAudioTrack(AUDIO_TRACK_ID, audioSource) + localAudioTrack?.setEnabled(true) localViewRenderer.setMirror(true) localVideoTrack?.addSink(localViewRenderer) @@ -334,27 +270,27 @@ class WebRtcPeerConnectionManager @Inject constructor( Timber.v("## VOIP Local video source created") videoCapturer.initialize(surfaceTextureHelper, context.applicationContext, videoSource!!.capturerObserver) videoCapturer.startCapture(1280, 720, 30) - localVideoTrack = peerConnectionFactory?.createVideoTrack("ARDAMSv0", videoSource)?.also { - Timber.v("## VOIP Local video track created") - localSurfaceRenderer?.get()?.let { surface -> - it.addSink(surface) - } + localVideoTrack = peerConnectionFactory?.createVideoTrack("ARDAMSv0", videoSource) + Timber.v("## VOIP Local video track created") + localSurfaceRenderer?.get()?.let { surface -> + localVideoTrack?.addSink(surface) } + localVideoTrack?.setEnabled(true) + + localVideoTrack?.addSink(localViewRenderer) localMediaStream?.addTrack(localVideoTrack) + remoteVideoTrack?.let { + it.setEnabled(true) + it.addSink(remoteViewRenderer) + } } - localVideoTrack?.addSink(localViewRenderer) - remoteVideoTrack?.let { - it.setEnabled(true) - it.addSink(remoteViewRenderer) - } - localMediaStream?.addTrack(audioTrack) + localMediaStream?.addTrack(localAudioTrack) Timber.v("## VOIP add local stream to peer connection") peerConnection?.addStream(localMediaStream) - localSurfaceRenderer = WeakReference(localViewRenderer) - remoteSurfaceRenderer = WeakReference(remoteViewRenderer) + startCall() } fun detachRenderers() { diff --git a/vector/src/main/java/im/vector/riotx/features/call/telecom/CallConnection.kt b/vector/src/main/java/im/vector/riotx/features/call/telecom/CallConnection.kt index 6f0c121b3d..6273abddfd 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/telecom/CallConnection.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/telecom/CallConnection.kt @@ -23,10 +23,6 @@ import android.telecom.DisconnectCause import androidx.annotation.RequiresApi import im.vector.riotx.features.call.VectorCallViewModel import im.vector.riotx.features.call.WebRtcPeerConnectionManager -import org.webrtc.IceCandidate -import org.webrtc.MediaStream -import org.webrtc.SessionDescription -import org.webrtc.VideoTrack import timber.log.Timber import javax.inject.Inject @@ -34,7 +30,7 @@ import javax.inject.Inject private val context: Context, private val roomId: String, val callId: String -) : Connection(), WebRtcPeerConnectionManager.Listener { +) : Connection() { @Inject lateinit var peerConnectionManager: WebRtcPeerConnectionManager @Inject lateinit var callViewModel: VectorCallViewModel @@ -113,20 +109,4 @@ import javax.inject.Inject //peerConnectionManager.startCall() */ } - - override fun addLocalIceCandidate(candidates: IceCandidate) { - } - - override fun addRemoteVideoTrack(videoTrack: VideoTrack) { - } - - override fun addLocalVideoTrack(videoTrack: VideoTrack) { - } - - override fun removeRemoteVideoStream(mediaStream: MediaStream) { - } - - override fun sendOffer(sessionDescription: SessionDescription) { - - } }