From 7939ecaedce33c93833bfaa954ece0b9a8871be8 Mon Sep 17 00:00:00 2001 From: Onuray Sahin Date: Fri, 22 Apr 2022 15:50:40 +0300 Subject: [PATCH] Try to start streaming screen capture. --- .../app/features/call/VectorCallActivity.kt | 35 +++++++++++++++---- .../features/call/VectorCallViewActions.kt | 3 +- .../app/features/call/VectorCallViewModel.kt | 2 +- .../webrtc/ScreenCaptureServiceConnection.kt | 10 +++++- .../app/features/call/webrtc/WebRtcCall.kt | 24 +++++++++++-- 5 files changed, 62 insertions(+), 12 deletions(-) diff --git a/vector/src/main/java/im/vector/app/features/call/VectorCallActivity.kt b/vector/src/main/java/im/vector/app/features/call/VectorCallActivity.kt index ea9adcde85..1ab423d541 100644 --- a/vector/src/main/java/im/vector/app/features/call/VectorCallActivity.kt +++ b/vector/src/main/java/im/vector/app/features/call/VectorCallActivity.kt @@ -24,6 +24,7 @@ import android.content.Intent import android.content.Intent.FLAG_ACTIVITY_CLEAR_TOP import android.content.res.Configuration import android.graphics.Color +import android.media.projection.MediaProjection import android.media.projection.MediaProjectionManager import android.os.Build import android.os.Bundle @@ -32,6 +33,7 @@ import android.util.Rational import android.view.MenuItem import android.view.View import android.view.WindowManager +import androidx.activity.result.ActivityResult import androidx.annotation.StringRes import androidx.core.content.ContextCompat import androidx.core.content.getSystemService @@ -76,6 +78,7 @@ import org.matrix.android.sdk.api.session.call.TurnServerResponse import org.matrix.android.sdk.api.session.room.model.call.EndCallReason import org.webrtc.EglBase import org.webrtc.RendererCommon +import org.webrtc.ScreenCapturerAndroid import timber.log.Timber import javax.inject.Inject @@ -636,18 +639,36 @@ class VectorCallActivity : VectorBaseActivity(), CallContro private val screenSharingPermissionActivityResultLauncher = registerStartForActivityResult { activityResult -> if (activityResult.resultCode == Activity.RESULT_OK) { - callViewModel.handle(VectorCallViewActions.StartScreenSharing) - // We need to start a foreground service with a sticky notification during screen sharing if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { - ContextCompat.startForegroundService( - this, - Intent(this, ScreenCaptureService::class.java) - ) - screenCaptureServiceConnection.bind() + // We need to start a foreground service with a sticky notification during screen sharing + startScreenSharingService(activityResult) + } else { + startScreenSharing(activityResult) } } } + private fun startScreenSharing(activityResult: ActivityResult) { + val videoCapturer = ScreenCapturerAndroid(activityResult.data, object : MediaProjection.Callback() { + override fun onStop() { + Timber.v("User revoked the screen capturing permission") + } + }) + callViewModel.handle(VectorCallViewActions.StartScreenSharing(videoCapturer)) + } + + private fun startScreenSharingService(activityResult: ActivityResult) { + ContextCompat.startForegroundService( + this, + Intent(this, ScreenCaptureService::class.java) + ) + screenCaptureServiceConnection.bind(object : ScreenCaptureServiceConnection.Callback { + override fun onServiceConnected() { + startScreenSharingService(activityResult) + } + }) + } + private fun handleShowScreenSharingPermissionDialog() { getSystemService()?.let { navigator.openScreenSharingPermissionDialog(it.createScreenCaptureIntent(), screenSharingPermissionActivityResultLauncher) diff --git a/vector/src/main/java/im/vector/app/features/call/VectorCallViewActions.kt b/vector/src/main/java/im/vector/app/features/call/VectorCallViewActions.kt index c84f733b9a..cec118f296 100644 --- a/vector/src/main/java/im/vector/app/features/call/VectorCallViewActions.kt +++ b/vector/src/main/java/im/vector/app/features/call/VectorCallViewActions.kt @@ -19,6 +19,7 @@ package im.vector.app.features.call import im.vector.app.core.platform.VectorViewModelAction import im.vector.app.features.call.audio.CallAudioManager import im.vector.app.features.call.transfer.CallTransferResult +import org.webrtc.VideoCapturer sealed class VectorCallViewActions : VectorViewModelAction { object EndCall : VectorCallViewActions() @@ -41,5 +42,5 @@ sealed class VectorCallViewActions : VectorViewModelAction { data class CallTransferSelectionResult(val callTransferResult: CallTransferResult) : VectorCallViewActions() object TransferCall : VectorCallViewActions() object ToggleScreenSharing : VectorCallViewActions() - object StartScreenSharing : VectorCallViewActions() + data class StartScreenSharing(val videoCapturer: VideoCapturer) : VectorCallViewActions() } diff --git a/vector/src/main/java/im/vector/app/features/call/VectorCallViewModel.kt b/vector/src/main/java/im/vector/app/features/call/VectorCallViewModel.kt index 3b7baef155..3b6ff9997b 100644 --- a/vector/src/main/java/im/vector/app/features/call/VectorCallViewModel.kt +++ b/vector/src/main/java/im/vector/app/features/call/VectorCallViewModel.kt @@ -348,7 +348,7 @@ class VectorCallViewModel @AssistedInject constructor( handleToggleScreenSharing(state.isSharingScreen) } is VectorCallViewActions.StartScreenSharing -> { - call?.startSharingScreen() + call?.startSharingScreen(action.videoCapturer) setState { copy(isSharingScreen = true) } diff --git a/vector/src/main/java/im/vector/app/features/call/webrtc/ScreenCaptureServiceConnection.kt b/vector/src/main/java/im/vector/app/features/call/webrtc/ScreenCaptureServiceConnection.kt index 922e9676a8..b8d28791b5 100644 --- a/vector/src/main/java/im/vector/app/features/call/webrtc/ScreenCaptureServiceConnection.kt +++ b/vector/src/main/java/im/vector/app/features/call/webrtc/ScreenCaptureServiceConnection.kt @@ -27,10 +27,17 @@ class ScreenCaptureServiceConnection @Inject constructor( private val context: Context ) : ServiceConnection { + interface Callback { + fun onServiceConnected() + } + private var isBound = false private var screenCaptureService: ScreenCaptureService? = null + private var callback: Callback? = null + + fun bind(callback: Callback) { + this.callback = callback - fun bind() { if (!isBound) { Intent(context, ScreenCaptureService::class.java).also { intent -> context.bindService(intent, this, 0) @@ -45,6 +52,7 @@ class ScreenCaptureServiceConnection @Inject constructor( override fun onServiceConnected(className: ComponentName, binder: IBinder) { screenCaptureService = (binder as ScreenCaptureService.LocalBinder).getService() isBound = true + callback?.onServiceConnected() } override fun onServiceDisconnected(className: ComponentName) { diff --git a/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCall.kt b/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCall.kt index bc8ae51a88..aac5ecc962 100644 --- a/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCall.kt +++ b/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCall.kt @@ -84,6 +84,7 @@ import org.webrtc.RtpTransceiver import org.webrtc.SessionDescription import org.webrtc.SurfaceTextureHelper import org.webrtc.SurfaceViewRenderer +import org.webrtc.VideoCapturer import org.webrtc.VideoSource import org.webrtc.VideoTrack import timber.log.Timber @@ -770,8 +771,27 @@ class WebRtcCall( return currentCaptureFormat } - fun startSharingScreen() { - // TODO. Will be handled within the next PR. + fun startSharingScreen(videoCapturer: VideoCapturer) { + val factory = peerConnectionFactoryProvider.get() ?: return + val videoSource = factory.createVideoSource(true) + val audioSource = factory.createAudioSource(DEFAULT_AUDIO_CONSTRAINTS) + val surfaceTextureHelper = SurfaceTextureHelper.create("CaptureThread", rootEglBase!!.eglBaseContext) + videoCapturer.initialize(surfaceTextureHelper, context, videoSource.capturerObserver) + videoCapturer.startCapture(currentCaptureFormat.width, currentCaptureFormat.height, currentCaptureFormat.fps) + + val videoTrack = factory.createVideoTrack("ARDAMSv0", videoSource).apply { setEnabled(true) } + val audioTrack = factory.createAudioTrack("ARDAMSa0", audioSource).apply { setEnabled(true) } + + val localMediaStream = factory.createLocalMediaStream("ARDAMS") + peerConnection?.addTrack(videoTrack) + peerConnection?.addTrack(audioTrack) + localMediaStream.addTrack(videoTrack) + localMediaStream.addTrack(audioTrack) + + localAudioSource = audioSource + localVideoSource = videoSource + localAudioTrack = audioTrack + localVideoTrack = videoTrack } fun stopSharingScreen() {