From eabb0bb41de3c148deb772a4e1287e8cc1f32949 Mon Sep 17 00:00:00 2001 From: Valere Date: Mon, 15 Jun 2020 18:17:59 +0200 Subject: [PATCH] Restart capture when camera is back to available --- .../call/CameraEventsHandlerAdapter.kt | 46 +++++++++++++ .../call/SharedActiveCallViewModel.kt | 4 ++ .../riotx/features/call/VectorCallActivity.kt | 1 + .../features/call/VectorCallViewModel.kt | 15 +++++ .../call/WebRtcPeerConnectionManager.kt | 64 ++++++++++++++++--- 5 files changed, 122 insertions(+), 8 deletions(-) create mode 100644 vector/src/main/java/im/vector/riotx/features/call/CameraEventsHandlerAdapter.kt diff --git a/vector/src/main/java/im/vector/riotx/features/call/CameraEventsHandlerAdapter.kt b/vector/src/main/java/im/vector/riotx/features/call/CameraEventsHandlerAdapter.kt new file mode 100644 index 0000000000..48f4b9b27b --- /dev/null +++ b/vector/src/main/java/im/vector/riotx/features/call/CameraEventsHandlerAdapter.kt @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.riotx.features.call + +import org.webrtc.CameraVideoCapturer +import timber.log.Timber + +open class CameraEventsHandlerAdapter : CameraVideoCapturer.CameraEventsHandler { + override fun onCameraError(p0: String?) { + Timber.v("## VOIP onCameraError $p0") + } + + override fun onCameraOpening(p0: String?) { + Timber.v("## VOIP onCameraOpening $p0") + } + + override fun onCameraDisconnected() { + Timber.v("## VOIP onCameraOpening") + } + + override fun onCameraFreezed(p0: String?) { + Timber.v("## VOIP onCameraFreezed $p0") + } + + override fun onFirstFrameAvailable() { + Timber.v("## VOIP onFirstFrameAvailable") + } + + override fun onCameraClosed() { + Timber.v("## VOIP onCameraClosed") + } +} diff --git a/vector/src/main/java/im/vector/riotx/features/call/SharedActiveCallViewModel.kt b/vector/src/main/java/im/vector/riotx/features/call/SharedActiveCallViewModel.kt index efd8541e1c..e76b690a23 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/SharedActiveCallViewModel.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/SharedActiveCallViewModel.kt @@ -37,6 +37,10 @@ class SharedActiveCallViewModel @Inject constructor( override fun onCurrentCallChange(call: MxCall?) { activeCall.postValue(call) } + + override fun onCaptureStateChanged(captureInError: Boolean) { + // nop + } } init { diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt index 5bb1d8d93c..256592656b 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt @@ -264,6 +264,7 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis if (callArgs.isVideoCall) { callVideoGroup.isVisible = true callInfoGroup.isVisible = false + pip_video_view.isVisible = !state.isVideoCaptureInError } else { callVideoGroup.isInvisible = true callInfoGroup.isVisible = true diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt index 809df48517..47888dfe76 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt @@ -41,6 +41,7 @@ data class VectorCallViewState( val isVideoCall: Boolean, val isAudioMuted: Boolean = false, val isVideoEnabled: Boolean = true, + val isVideoCaptureInError: Boolean = false, val soundDevice: CallAudioManager.SoundDevice = CallAudioManager.SoundDevice.PHONE, val otherUserMatrixItem: Async = Uninitialized, val callState: Async = Uninitialized @@ -83,12 +84,25 @@ class VectorCallViewModel @AssistedInject constructor( } } + private val currentCallListener = object : WebRtcPeerConnectionManager.CurrentCallListener { + override fun onCurrentCallChange(call: MxCall?) { + } + + override fun onCaptureStateChanged(captureInError: Boolean) { + setState { + copy(isVideoCaptureInError = captureInError) + } + } + } + init { autoReplyIfNeeded = args.autoAccept initialState.callId?.let { + webRtcPeerConnectionManager.addCurrentCallListener(currentCallListener) + session.callSignalingService().getCallWithId(it)?.let { mxCall -> this.call = mxCall mxCall.otherUserId @@ -109,6 +123,7 @@ class VectorCallViewModel @AssistedInject constructor( override fun onCleared() { // session.callService().removeCallListener(callServiceListener) + webRtcPeerConnectionManager.removeCurrentCallListener(currentCallListener) this.call?.removeListener(callStateListener) super.onCleared() } diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt index f40d02d2f8..9cbd0dfddd 100644 --- a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -17,6 +17,9 @@ package im.vector.riotx.features.call import android.content.Context +import android.hardware.camera2.CameraManager +import android.os.Build +import androidx.annotation.RequiresApi import im.vector.matrix.android.api.MatrixCallback import im.vector.matrix.android.api.extensions.tryThis import im.vector.matrix.android.api.session.call.CallState @@ -71,6 +74,7 @@ class WebRtcPeerConnectionManager @Inject constructor( interface CurrentCallListener { fun onCurrentCallChange(call: MxCall?) + fun onCaptureStateChanged(captureInError: Boolean) } private val currentCallsListeners = emptyList().toMutableList() @@ -118,6 +122,9 @@ class WebRtcPeerConnectionManager @Inject constructor( var remoteCandidateSource: ReplaySubject? = null var remoteIceCandidateDisposable: Disposable? = null + // We register an availability callback if we loose access to camera + var cameraAvailabilityCallback: CameraRestarter? = null + fun release() { remoteIceCandidateDisposable?.dispose() iceCandidateDisposable?.dispose() @@ -149,6 +156,14 @@ class WebRtcPeerConnectionManager @Inject constructor( private var videoCapturer: VideoCapturer? = null + var capturerIsInError = false + set(value) { + field = value + currentCallsListeners.forEach { + tryThis { it.onCaptureStateChanged(value) } + } + } + var localSurfaceRenderer: WeakReference? = null var remoteSurfaceRenderer: WeakReference? = null @@ -355,7 +370,27 @@ class WebRtcPeerConnectionManager @Inject constructor( ?.firstOrNull { cameraIterator.isFrontFacing(it) } ?: cameraIterator.deviceNames?.first() - val videoCapturer = cameraIterator.createCapturer(frontCamera, null) + // TODO detect when no camera or no front camera + + val videoCapturer = cameraIterator.createCapturer(frontCamera, object : CameraEventsHandlerAdapter() { + override fun onFirstFrameAvailable() { + super.onFirstFrameAvailable() + capturerIsInError = false + } + + override fun onCameraClosed() { + // This could happen if you open the camera app in chat + // We then register in order to restart capture as soon as the camera is available again + Timber.v("## VOIP onCameraClosed") + this@WebRtcPeerConnectionManager.capturerIsInError = true + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { + val restarter = CameraRestarter(frontCamera ?: "", callContext.mxCall.callId) + callContext.cameraAvailabilityCallback = restarter + val cameraManager = context.getSystemService(Context.CAMERA_SERVICE) as CameraManager + cameraManager.registerAvailabilityCallback(restarter, null) + } + } + }) val videoSource = peerConnectionFactory!!.createVideoSource(videoCapturer.isScreencast) val surfaceTextureHelper = SurfaceTextureHelper.create("CaptureThread", rootEglBase!!.eglBaseContext) @@ -374,14 +409,7 @@ class WebRtcPeerConnectionManager @Inject constructor( callContext.localVideoSource = videoSource callContext.localVideoTrack = localVideoTrack -// localViewRenderer?.let { localVideoTrack?.addSink(it) } localMediaStream?.addTrack(localVideoTrack) -// callContext.localMediaStream = localMediaStream -// remoteVideoTrack?.setEnabled(true) -// remoteVideoTrack?.let { -// it.setEnabled(true) -// it.addSink(remoteViewRenderer) -// } } } @@ -542,6 +570,12 @@ class WebRtcPeerConnectionManager @Inject constructor( } fun endCall() { + currentCall?.cameraAvailabilityCallback?.let { cameraAvailabilityCallback -> + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { + val cameraManager = context.getSystemService(Context.CAMERA_SERVICE) as CameraManager + cameraManager.unregisterAvailabilityCallback(cameraAvailabilityCallback) + } + } currentCall?.mxCall?.hangUp() currentCall = null audioManager.stop() @@ -746,4 +780,18 @@ class WebRtcPeerConnectionManager @Inject constructor( Timber.v("## VOIP StreamObserver onAddTrack") } } + + @RequiresApi(Build.VERSION_CODES.LOLLIPOP) + inner class CameraRestarter(val cameraId: String, val callId: String) : CameraManager.AvailabilityCallback() { + + override fun onCameraAvailable(cameraId: String) { + if (this.cameraId == cameraId && currentCall?.mxCall?.callId == callId) { + // re-start the capture + // TODO notify that video is enabled + videoCapturer?.startCapture(1280, 720, 30) + (context.getSystemService(Context.CAMERA_SERVICE) as? CameraManager) + ?.unregisterAvailabilityCallback(this) + } + } + } }