filippodb
/
SchildiChat-android
mirror of https://github.com/SchildiChat/SchildiChat-android.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

VoIP: allow hold/resume from sdk (activate unified plan semantics)

This commit is contained in:
ganfra 2020-11-19 20:44:33 +01:00
parent 7d63135cc2
commit f960cf2ce9
1 changed files with 144 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

261
vector/src/main/java/im/vector/app/features/call/WebRtcPeerConnectionManager.kt
View File

@ -71,6 +71,7 @@ import org.webrtc.IceCandidate
import org.webrtc.MediaConstraints import org.webrtc.MediaConstraints
import org.webrtc.MediaStream import org.webrtc.MediaStream
import org.webrtc.PeerConnection import org.webrtc.PeerConnection
import org.webrtc.PeerConnection.RTCConfiguration
import org.webrtc.PeerConnectionFactory import org.webrtc.PeerConnectionFactory
import org.webrtc.RtpReceiver import org.webrtc.RtpReceiver
import org.webrtc.RtpTransceiver import org.webrtc.RtpTransceiver
@ -121,26 +122,23 @@ class WebRtcPeerConnectionManager @Inject constructor(
} }
} }
data class CallContext( class CallContext(val mxCall: MxCall) {
val mxCall: MxCall,
var peerConnection: PeerConnection? = null, var peerConnection: PeerConnection? = null
var localAudioSource: AudioSource? = null
var localAudioTrack: AudioTrack? = null
var localVideoSource: VideoSource? = null
var localVideoTrack: VideoTrack? = null
var remoteVideoTrack: VideoTrack? = null
var localMediaStream: MediaStream? = null, // Perfect negotiation state: https://www.w3.org/TR/webrtc/#perfect-negotiation-example
var remoteMediaStream: MediaStream? = null, var makingOffer: Boolean = false
var ignoreOffer: Boolean = false
var localAudioSource: AudioSource? = null, // Mute status
var localAudioTrack: AudioTrack? = null, var micMuted = false
var videoMuted = false
var localVideoSource: VideoSource? = null, var remoteOnHold = false
var localVideoTrack: VideoTrack? = null,
var remoteVideoTrack: VideoTrack? = null,
// Perfect negotiation state: https://www.w3.org/TR/webrtc/#perfect-negotiation-example
var makingOffer: Boolean = false,
var ignoreOffer: Boolean = false
) {
var offerSdp: CallInviteContent.Offer? = null var offerSdp: CallInviteContent.Offer? = null
@ -176,8 +174,6 @@ class WebRtcPeerConnectionManager @Inject constructor(
localAudioTrack = null localAudioTrack = null
localVideoSource = null localVideoSource = null
localVideoTrack = null localVideoTrack = null
localMediaStream = null
remoteMediaStream = null
} }
} }
@ -207,27 +203,24 @@ class WebRtcPeerConnectionManager @Inject constructor(
} }
} }
var localSurfaceRenderer: MutableList<WeakReference<SurfaceViewRenderer>> = ArrayList() var localSurfaceRenderers: MutableList<WeakReference<SurfaceViewRenderer>> = ArrayList()
var remoteSurfaceRenderer: MutableList<WeakReference<SurfaceViewRenderer>> = ArrayList() var remoteSurfaceRenderers: MutableList<WeakReference<SurfaceViewRenderer>> = ArrayList()
fun addIfNeeded(renderer: SurfaceViewRenderer?, list: MutableList<WeakReference<SurfaceViewRenderer>>) { private fun MutableList<WeakReference<SurfaceViewRenderer>>.addIfNeeded(renderer: SurfaceViewRenderer?) {
if (renderer == null) return if (renderer == null) return
val exists = list.firstOrNull { val exists = any {
it.get() == renderer it.get() == renderer
} != null }
if (!exists) { if (!exists) {
list.add(WeakReference(renderer)) add(WeakReference(renderer))
} }
} }
fun removeIfNeeded(renderer: SurfaceViewRenderer?, list: MutableList<WeakReference<SurfaceViewRenderer>>) { private fun MutableList<WeakReference<SurfaceViewRenderer>>.removeIfNeeded(renderer: SurfaceViewRenderer?) {
if (renderer == null) return if (renderer == null) return
val exists = list.indexOfFirst { removeAll {
it.get() == renderer it.get() == renderer
} }
if (exists != -1) {
list.add(WeakReference(renderer))
}
} }
@OnLifecycleEvent(Lifecycle.Event.ON_RESUME) @OnLifecycleEvent(Lifecycle.Event.ON_RESUME)
@ -308,7 +301,10 @@ class WebRtcPeerConnectionManager @Inject constructor(
} }
} }
Timber.v("## VOIP creating peer connection...with iceServers $iceServers ") Timber.v("## VOIP creating peer connection...with iceServers $iceServers ")
callContext.peerConnection = peerConnectionFactory?.createPeerConnection(iceServers, StreamObserver(callContext)) val rtcConfig = RTCConfiguration(iceServers).apply {
sdpSemantics = PeerConnection.SdpSemantics.UNIFIED_PLAN
}
callContext.peerConnection = peerConnectionFactory?.createPeerConnection(rtcConfig, StreamObserver(callContext))
} }
private fun CoroutineScope.sendSdpOffer(callContext: CallContext) = launch(dispatcher) { private fun CoroutineScope.sendSdpOffer(callContext: CallContext) = launch(dispatcher) {
@ -357,8 +353,8 @@ class WebRtcPeerConnectionManager @Inject constructor(
Timber.v("## VOIP attachViewRenderers localRendeder $localViewRenderer / $remoteViewRenderer") Timber.v("## VOIP attachViewRenderers localRendeder $localViewRenderer / $remoteViewRenderer")
// this.localSurfaceRenderer = WeakReference(localViewRenderer) // this.localSurfaceRenderer = WeakReference(localViewRenderer)
// this.remoteSurfaceRenderer = WeakReference(remoteViewRenderer) // this.remoteSurfaceRenderer = WeakReference(remoteViewRenderer)
addIfNeeded(localViewRenderer, this.localSurfaceRenderer) localSurfaceRenderers.addIfNeeded(localViewRenderer)
addIfNeeded(remoteViewRenderer, this.remoteSurfaceRenderer) remoteSurfaceRenderers.addIfNeeded(remoteViewRenderer)
// The call is going to resume from background, we can reduce notif // The call is going to resume from background, we can reduce notif
currentCall?.mxCall currentCall?.mxCall
@ -388,34 +384,36 @@ class WebRtcPeerConnectionManager @Inject constructor(
// TODO eventually we could already display local stream in PIP? // TODO eventually we could already display local stream in PIP?
} }
VectorCallActivity.OUTGOING_CREATED -> { VectorCallActivity.OUTGOING_CREATED -> {
call.mxCall.state = CallState.CreateOffer internalSetupOutgoingCall(call, turnServer)
// 1. Create RTCPeerConnection
createPeerConnection(call, turnServer)
// 2. Access camera (if video call) + microphone, create local stream
createLocalStream(call)
// 3. add local stream
call.localMediaStream?.let { call.peerConnection?.addStream(it) }
attachViewRenderersInternal()
Timber.v("## VOIP remoteCandidateSource ${call.remoteCandidateSource}")
call.remoteIceCandidateDisposable = call.remoteCandidateSource?.subscribe({
Timber.v("## VOIP adding remote ice candidate $it")
call.peerConnection?.addIceCandidate(it)
}, {
Timber.v("## VOIP failed to add remote ice candidate $it")
})
// Now wait for negotiation callback
} }
else -> { else -> {
// sink existing tracks (configuration change, e.g screen rotation) // sink existing tracks (configuration change, e.g screen rotation)
attachViewRenderersInternal() attachViewRenderersInternal(call)
} }
} }
} }
} }
private suspend fun internalSetupOutgoingCall(call: CallContext, turnServer: TurnServerResponse?) {
call.mxCall.state = CallState.CreateOffer
// 1. Create RTCPeerConnection
createPeerConnection(call, turnServer)
// 2. Access camera (if video call) + microphone, create local stream
createLocalStream(call)
attachViewRenderersInternal(call)
Timber.v("## VOIP remoteCandidateSource ${call.remoteCandidateSource}")
call.remoteIceCandidateDisposable = call.remoteCandidateSource?.subscribe({
Timber.v("## VOIP adding remote ice candidate $it")
call.peerConnection?.addIceCandidate(it)
}, {
Timber.v("## VOIP failed to add remote ice candidate $it")
})
// Now wait for negotiation callback
}
private suspend fun internalAcceptIncomingCall(callContext: CallContext, turnServerResponse: TurnServerResponse?) { private suspend fun internalAcceptIncomingCall(callContext: CallContext, turnServerResponse: TurnServerResponse?) {
val mxCall = callContext.mxCall val mxCall = callContext.mxCall
// Update service state // Update service state
@ -436,20 +434,27 @@ class WebRtcPeerConnectionManager @Inject constructor(
// create sdp using offer, and set remote description // create sdp using offer, and set remote description
// the offer has beed stored when invite was received // the offer has beed stored when invite was received
callContext.offerSdp?.sdp?.let { val offerSdp = callContext.offerSdp?.sdp?.let {
SessionDescription(SessionDescription.Type.OFFER, it) SessionDescription(SessionDescription.Type.OFFER, it)
}?.let { }
callContext.peerConnection?.setRemoteDescription(SdpObserverAdapter(), it) if (offerSdp == null) {
Timber.v("We don't have any offer to process")
return
}
Timber.v("Offer sdp for invite: ${offerSdp.description}")
try {
callContext.peerConnection?.awaitSetRemoteDescription(offerSdp)
} catch (failure: Throwable) {
Timber.v("Failure putting remote description")
return
} }
// 2) Access camera + microphone, create local stream // 2) Access camera + microphone, create local stream
createLocalStream(callContext) createLocalStream(callContext)
// 2) add local stream attachViewRenderersInternal(callContext)
currentCall?.localMediaStream?.let { callContext.peerConnection?.addStream(it) }
attachViewRenderersInternal()
// create a answer, set local description and send via signaling // create a answer, set local description and send via signaling
createAnswer()?.also { createAnswer(callContext)?.also {
callContext.mxCall.accept(it) callContext.mxCall.accept(it)
} }
Timber.v("## VOIP remoteCandidateSource ${callContext.remoteCandidateSource}") Timber.v("## VOIP remoteCandidateSource ${callContext.remoteCandidateSource}")
@ -462,28 +467,20 @@ class WebRtcPeerConnectionManager @Inject constructor(
} }
private fun createLocalStream(callContext: CallContext) { private fun createLocalStream(callContext: CallContext) {
if (callContext.localMediaStream != null) {
Timber.e("## VOIP localMediaStream already created")
return
}
if (peerConnectionFactory == null) { if (peerConnectionFactory == null) {
Timber.e("## VOIP peerConnectionFactory is null") Timber.e("## VOIP peerConnectionFactory is null")
return return
} }
Timber.v("Create local stream for call ${callContext.mxCall.callId}")
val audioSource = peerConnectionFactory!!.createAudioSource(DEFAULT_AUDIO_CONSTRAINTS) val audioSource = peerConnectionFactory!!.createAudioSource(DEFAULT_AUDIO_CONSTRAINTS)
val localAudioTrack = peerConnectionFactory!!.createAudioTrack(AUDIO_TRACK_ID, audioSource) val audioTrack = peerConnectionFactory!!.createAudioTrack(AUDIO_TRACK_ID, audioSource)
localAudioTrack?.setEnabled(true) audioTrack.setEnabled(true)
Timber.v("Add audio track $AUDIO_TRACK_ID to call ${callContext.mxCall.callId}")
callContext.localAudioSource = audioSource callContext.apply {
callContext.localAudioTrack = localAudioTrack peerConnection?.addTrack(audioTrack, listOf(STREAM_ID))
localAudioSource = audioSource
val localMediaStream = peerConnectionFactory!!.createLocalMediaStream("ARDAMS") // magic value? localAudioTrack = audioTrack
}
// Add audio track
localMediaStream?.addTrack(localAudioTrack)
callContext.localMediaStream = localMediaStream
// add video track if needed // add video track if needed
if (callContext.mxCall.isVideoCall) { if (callContext.mxCall.isVideoCall) {
availableCamera.clear() availableCamera.clear()
@ -535,35 +532,33 @@ class WebRtcPeerConnectionManager @Inject constructor(
videoCapturer.startCapture(currentCaptureMode.width, currentCaptureMode.height, currentCaptureMode.fps) videoCapturer.startCapture(currentCaptureMode.width, currentCaptureMode.height, currentCaptureMode.fps)
this.videoCapturer = videoCapturer this.videoCapturer = videoCapturer
val localVideoTrack = peerConnectionFactory!!.createVideoTrack("ARDAMSv0", videoSource) val videoTrack = peerConnectionFactory!!.createVideoTrack(VIDEO_TRACK_ID, videoSource)
Timber.v("## VOIP Local video track created") Timber.v("Add video track $VIDEO_TRACK_ID to call ${callContext.mxCall.callId}")
localVideoTrack?.setEnabled(true) videoTrack.setEnabled(true)
callContext.apply {
callContext.localVideoSource = videoSource peerConnection?.addTrack(videoTrack, listOf(STREAM_ID))
callContext.localVideoTrack = localVideoTrack localVideoSource = videoSource
localVideoTrack = videoTrack
localMediaStream?.addTrack(localVideoTrack) }
} }
} }
updateMuteStatus(callContext)
} }
private fun attachViewRenderersInternal() { private fun attachViewRenderersInternal(call: CallContext) {
// render local video in pip view // render local video in pip view
localSurfaceRenderer.forEach { localSurfaceRenderers.forEach { renderer ->
it.get()?.let { pipSurface -> renderer.get()?.let { pipSurface ->
pipSurface.setMirror(this.cameraInUse?.type == CameraType.FRONT) pipSurface.setMirror(this.cameraInUse?.type == CameraType.FRONT)
// no need to check if already added, addSink is checking that // no need to check if already added, addSink is checking that
currentCall?.localVideoTrack?.addSink(pipSurface) call.localVideoTrack?.addSink(pipSurface)
} }
} }
// If remote track exists, then sink it to surface // If remote track exists, then sink it to surface
remoteSurfaceRenderer.forEach { remoteSurfaceRenderers.forEach { renderer ->
it.get()?.let { participantSurface -> renderer.get()?.let { participantSurface ->
currentCall?.remoteVideoTrack?.let { call.remoteVideoTrack?.addSink(participantSurface)
// no need to check if already added, addSink is checking that
it.addSink(participantSurface)
}
} }
} }
} }
@ -579,30 +574,30 @@ class WebRtcPeerConnectionManager @Inject constructor(
} }
} }
fun detachRenderers(renderes: List<SurfaceViewRenderer>?) { fun detachRenderers(renderers: List<SurfaceViewRenderer>?) {
Timber.v("## VOIP detachRenderers") Timber.v("## VOIP detachRenderers")
// currentCall?.localMediaStream?.let { currentCall?.peerConnection?.removeStream(it) } // currentCall?.localMediaStream?.let { currentCall?.peerConnection?.removeStream(it) }
if (renderes.isNullOrEmpty()) { if (renderers.isNullOrEmpty()) {
// remove all sinks // remove all sinks
localSurfaceRenderer.forEach { localSurfaceRenderers.forEach {
if (it.get() != null) currentCall?.localVideoTrack?.removeSink(it.get()) if (it.get() != null) currentCall?.localVideoTrack?.removeSink(it.get())
} }
remoteSurfaceRenderer.forEach { remoteSurfaceRenderers.forEach {
if (it.get() != null) currentCall?.remoteVideoTrack?.removeSink(it.get()) if (it.get() != null) currentCall?.remoteVideoTrack?.removeSink(it.get())
} }
localSurfaceRenderer.clear() localSurfaceRenderers.clear()
remoteSurfaceRenderer.clear() remoteSurfaceRenderers.clear()
} else { } else {
renderes.forEach { renderers.forEach {
removeIfNeeded(it, localSurfaceRenderer) localSurfaceRenderers.removeIfNeeded(it)
removeIfNeeded(it, remoteSurfaceRenderer) remoteSurfaceRenderers.removeIfNeeded(it)
// no need to check if it's in the track, removeSink is doing it // no need to check if it's in the track, removeSink is doing it
currentCall?.localVideoTrack?.removeSink(it) currentCall?.localVideoTrack?.removeSink(it)
currentCall?.remoteVideoTrack?.removeSink(it) currentCall?.remoteVideoTrack?.removeSink(it)
} }
} }
if (remoteSurfaceRenderer.isEmpty()) { if (remoteSurfaceRenderers.isEmpty()) {
// The call is going to continue in background, so ensure notification is visible // The call is going to continue in background, so ensure notification is visible
currentCall?.mxCall currentCall?.mxCall
?.takeIf { it.state is CallState.Connected } ?.takeIf { it.state is CallState.Connected }
@ -648,7 +643,9 @@ class WebRtcPeerConnectionManager @Inject constructor(
companion object { companion object {
private const val STREAM_ID = "ARDAMS"
private const val AUDIO_TRACK_ID = "ARDAMSa0" private const val AUDIO_TRACK_ID = "ARDAMSa0"
private const val VIDEO_TRACK_ID = "ARDAMSv0"
private val DEFAULT_AUDIO_CONSTRAINTS = MediaConstraints().apply { private val DEFAULT_AUDIO_CONSTRAINTS = MediaConstraints().apply {
// add all existing audio filters to avoid having echos // add all existing audio filters to avoid having echos
@ -765,9 +762,8 @@ class WebRtcPeerConnectionManager @Inject constructor(
} }
} }
private suspend fun createAnswer(): SessionDescription? { private suspend fun createAnswer(call: CallContext): SessionDescription? {
Timber.w("## VOIP createAnswer") Timber.w("## VOIP createAnswer")
val call = currentCall ?: return null
val peerConnection = call.peerConnection ?: return null val peerConnection = call.peerConnection ?: return null
val constraints = MediaConstraints().apply { val constraints = MediaConstraints().apply {
mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")) mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true"))
@ -784,11 +780,15 @@ class WebRtcPeerConnectionManager @Inject constructor(
} }
fun muteCall(muted: Boolean) { fun muteCall(muted: Boolean) {
currentCall?.localAudioTrack?.setEnabled(!muted) val call = currentCall ?: return
call.micMuted = muted
updateMuteStatus(call)
} }
fun enableVideo(enabled: Boolean) { fun enableVideo(enabled: Boolean) {
currentCall?.localVideoTrack?.setEnabled(enabled) val call = currentCall ?: return
call.videoMuted = !enabled
updateMuteStatus(call)
} }
fun switchCamera() { fun switchCamera() {
@ -800,7 +800,7 @@ class WebRtcPeerConnectionManager @Inject constructor(
override fun onCameraSwitchDone(isFrontCamera: Boolean) { override fun onCameraSwitchDone(isFrontCamera: Boolean) {
Timber.v("## VOIP onCameraSwitchDone isFront $isFrontCamera") Timber.v("## VOIP onCameraSwitchDone isFront $isFrontCamera")
cameraInUse = availableCamera.first { if (isFrontCamera) it.type == CameraType.FRONT else it.type == CameraType.BACK } cameraInUse = availableCamera.first { if (isFrontCamera) it.type == CameraType.FRONT else it.type == CameraType.BACK }
localSurfaceRenderer.forEach { localSurfaceRenderers.forEach {
it.get()?.setMirror(isFrontCamera) it.get()?.setMirror(isFrontCamera)
} }
@ -968,8 +968,7 @@ class WebRtcPeerConnectionManager @Inject constructor(
val sdp = SessionDescription(type.asWebRTC(), sdpText) val sdp = SessionDescription(type.asWebRTC(), sdpText)
peerConnection.awaitSetRemoteDescription(sdp) peerConnection.awaitSetRemoteDescription(sdp)
if (type == SdpType.OFFER) { if (type == SdpType.OFFER) {
// create a answer, set local description and send via signaling createAnswer(call)?.also {
createAnswer()?.also {
call.mxCall.negotiate(it) call.mxCall.negotiate(it)
} }
} }
@ -1003,12 +1002,13 @@ class WebRtcPeerConnectionManager @Inject constructor(
* rather than 'sendonly') * rather than 'sendonly')
* @returns true if the other party has put us on hold * @returns true if the other party has put us on hold
*/ */
private fun isLocalOnHold(callContext: CallContext): Boolean { fun isLocalOnHold(): Boolean {
if (callContext.mxCall.state !is CallState.Connected) return false val call = currentCall ?: return false
if (call.mxCall.state !is CallState.Connected) return false
var callOnHold = true var callOnHold = true
// We consider a call to be on hold only if *all* the tracks are on hold // We consider a call to be on hold only if *all* the tracks are on hold
// (is this the right thing to do?) // (is this the right thing to do?)
for (transceiver in callContext.peerConnection?.transceivers ?: emptyList()) { for (transceiver in call.peerConnection?.transceivers ?: emptyList()) {
val trackOnHold = transceiver.currentDirection == RtpTransceiver.RtpTransceiverDirection.INACTIVE val trackOnHold = transceiver.currentDirection == RtpTransceiver.RtpTransceiverDirection.INACTIVE
|| transceiver.currentDirection == RtpTransceiver.RtpTransceiverDirection.RECV_ONLY || transceiver.currentDirection == RtpTransceiver.RtpTransceiverDirection.RECV_ONLY
if (!trackOnHold) callOnHold = false; if (!trackOnHold) callOnHold = false;
@ -1016,6 +1016,33 @@ class WebRtcPeerConnectionManager @Inject constructor(
return callOnHold; return callOnHold;
} }
fun isRemoteOnHold(): Boolean {
val call = currentCall ?: return false
return call.remoteOnHold;
}
fun setRemoteOnHold(onHold: Boolean) {
val call = currentCall ?: return
if (call.remoteOnHold == onHold) return
call.remoteOnHold = onHold
val direction = if (onHold) {
RtpTransceiver.RtpTransceiverDirection.INACTIVE
} else {
RtpTransceiver.RtpTransceiverDirection.SEND_RECV
}
for (transceiver in call.peerConnection?.transceivers ?: emptyList()) {
transceiver.direction = direction
}
updateMuteStatus(call)
}
private fun updateMuteStatus(call: CallContext) {
val micShouldBeMuted = call.micMuted || call.remoteOnHold
call.localAudioTrack?.setEnabled(!micShouldBeMuted)
val vidShouldBeMuted = call.videoMuted || call.remoteOnHold
call.localVideoTrack?.setEnabled(!vidShouldBeMuted)
}
private inner class StreamObserver(val callContext: CallContext) : PeerConnection.Observer { private inner class StreamObserver(val callContext: CallContext) : PeerConnection.Observer {
override fun onConnectionChange(newState: PeerConnection.PeerConnectionState?) { override fun onConnectionChange(newState: PeerConnection.PeerConnectionState?) {
@ -1153,7 +1180,7 @@ class WebRtcPeerConnectionManager @Inject constructor(
remoteVideoTrack.setEnabled(true) remoteVideoTrack.setEnabled(true)
callContext.remoteVideoTrack = remoteVideoTrack callContext.remoteVideoTrack = remoteVideoTrack
// sink to renderer if attached // sink to renderer if attached
remoteSurfaceRenderer.forEach { it.get()?.let { remoteVideoTrack.addSink(it) } } remoteSurfaceRenderers.forEach { it.get()?.let { remoteVideoTrack.addSink(it) } }
} }
} }
} }
@ -1164,7 +1191,7 @@ class WebRtcPeerConnectionManager @Inject constructor(
// remoteSurfaceRenderer?.get()?.let { // remoteSurfaceRenderer?.get()?.let {
// callContext.remoteVideoTrack?.removeSink(it) // callContext.remoteVideoTrack?.removeSink(it)
// } // }
remoteSurfaceRenderer remoteSurfaceRenderers
.mapNotNull { it.get() } .mapNotNull { it.get() }
.forEach { callContext.remoteVideoTrack?.removeSink(it) } .forEach { callContext.remoteVideoTrack?.removeSink(it) }
callContext.remoteVideoTrack = null callContext.remoteVideoTrack = null