Make possible to select input media devices during calls

src/components/MediaDevicesPreview.vue

@@ -126,7 +126,7 @@ export default {
 				return mediaDevicesManager.attributes.audioInputId
 			},
 			set(value) {
-				mediaDevicesManager.attributes.audioInputId = value
+				mediaDevicesManager.set('audioInputId', value)
 			},
 		},
 
@@ -135,7 +135,7 @@ export default {
 				return mediaDevicesManager.attributes.videoInputId
 			},
 			set(value) {
-				mediaDevicesManager.attributes.videoInputId = value
+				mediaDevicesManager.set('videoInputId', value)
 			},
 		},
 

src/utils/webrtc/MediaDevicesManager.js

@@ -50,7 +50,11 @@
  * those cases the fallback label can be used instead.
  *
  * "attributes.audioInputId" and "attributes.videoInputId" define the devices
- * that will be used when calling "getUserMedia(constraints)".
+ * that will be used when calling "getUserMedia(constraints)". Clients of this
+ * class must modify them using "set('audioInputId', value)" and
+ * "set('videoInputId', value)" to ensure that change events are triggered.
+ * However, note that change events are not triggered when the devices are
+ * modified.
  *
  * The selected devices will be automatically cleared if they are no longer
  * available. When no device of certain kind is selected and there are other
@@ -67,17 +71,66 @@ export default function MediaDevicesManager() {
 		videoInputId: undefined,
 	}
 
+	this._handlers = []
+
 	this._enabledCount = 0
 
 	this._knownDevices = {}
 
 	this._fallbackAudioInputId = undefined
 	this._fallbackVideoInputId = undefined
 
+	this._tracks = []
+
 	this._updateDevicesBound = this._updateDevices.bind(this)
 }
 MediaDevicesManager.prototype = {
 
+	get: function(key) {
+		return this.attributes[key]
+	},
+
+	set: function(key, value) {
+		this.attributes[key] = value
+
+		this._trigger('change:' + key, [value])
+	},
+
+	on: function(event, handler) {
+		if (!this._handlers.hasOwnProperty(event)) {
+			this._handlers[event] = [handler]
+		} else {
+			this._handlers[event].push(handler)
+		}
+	},
+
+	off: function(event, handler) {
+		const handlers = this._handlers[event]
+		if (!handlers) {
+			return
+		}
+
+		const index = handlers.indexOf(handler)
+		if (index !== -1) {
+			handlers.splice(index, 1)
+		}
+	},
+
+	_trigger: function(event, args) {
+		let handlers = this._handlers[event]
+		if (!handlers) {
+			return
+		}
+
+		args.unshift(this)
+
+		handlers = handlers.slice(0)
+		for (let i = 0; i < handlers.length; i++) {
+			const handler = handlers[i]
+			handler.apply(handler, args)
+		}
+	},
+
 	/**
 	 * Returns whether getting user media and enumerating media devices is
 	 * supported or not.
@@ -119,6 +172,9 @@ MediaDevicesManager.prototype = {
 
 	_updateDevices: function() {
 		navigator.mediaDevices.enumerateDevices().then(devices => {
+			const previousAudioInputId = this.attributes.audioInputId
+			const previousVideoInputId = this.attributes.videoInputId
+
 			const removedDevices = this.attributes.devices.filter(oldDevice => !devices.find(device => oldDevice.deviceId === device.deviceId && oldDevice.kind === device.kind))
 			const updatedDevices = devices.filter(device => this.attributes.devices.find(oldDevice => device.deviceId === oldDevice.deviceId && device.kind === oldDevice.kind))
 			const addedDevices = devices.filter(device => !this.attributes.devices.find(oldDevice => device.deviceId === oldDevice.deviceId && device.kind === oldDevice.kind))
@@ -132,6 +188,15 @@ MediaDevicesManager.prototype = {
 			addedDevices.forEach(addedDevice => {
 				this._addDevice(addedDevice)
 			})
+
+			// Trigger change events after all the devices are processed to
+			// prevent change events for intermediate states.
+			if (previousAudioInputId !== this.attributes.audioInputId) {
+				this._trigger('change:audioInputId', [this.attributes.audioInputId])
+			}
+			if (previousVideoInputId !== this.attributes.videoInputId) {
+				this._trigger('change:videoInputId', [this.attributes.videoInputId])
+			}
 		}).catch(function(error) {
 			console.error('Could not update known media devices: ' + error.name + ': ' + error.message)
 		})
@@ -272,7 +337,11 @@ MediaDevicesManager.prototype = {
 			}
 		}
 
+		this._stopIncompatibleTracks(constraints)
+
 		return navigator.mediaDevices.getUserMedia(constraints).then(stream => {
+			this._registerStream(stream)
+
 			// In Firefox the dialog to grant media permissions allows the user
 			// to change the device to use, overriding the device that was
 			// originally requested.
@@ -294,14 +363,53 @@ MediaDevicesManager.prototype = {
 		})
 	},
 
+	_stopIncompatibleTracks: function(constraints) {
+		this._tracks.forEach(track => {
+			if (constraints.audio && constraints.audio.deviceId && track.kind === 'audio') {
+				const settings = track.getSettings()
+				if (settings && settings.deviceId !== constraints.audio.deviceId) {
+					track.stop()
+				}
+			}
+
+			if (constraints.video && constraints.video.deviceId && track.kind === 'video') {
+				const settings = track.getSettings()
+				if (settings && settings.deviceId !== constraints.video.deviceId) {
+					track.stop()
+				}
+			}
+		})
+	},
+
+	_registerStream: function(stream) {
+		stream.getTracks().forEach(track => {
+			this._registerTrack(track)
+		})
+	},
+
+	_registerTrack: function(track) {
+		this._tracks.push(track)
+
+		track.addEventListener('ended', () => {
+			const index = this._tracks.indexOf(track)
+			if (index >= 0) {
+				this._tracks.splice(index, 1)
+			}
+		})
+
+		track.addEventListener('cloned', event => {
+			this._registerTrack(event.detail)
+		})
+	},
+
 	_updateSelectedDevicesFromGetUserMediaResult: function(stream) {
 		if (this.attributes.audioInputId) {
 			const audioTracks = stream.getAudioTracks()
 			const audioTrackSettings = audioTracks.length > 0 ? audioTracks[0].getSettings() : null
 			if (audioTrackSettings && audioTrackSettings.deviceId && this.attributes.audioInputId !== audioTrackSettings.deviceId) {
 				console.debug('Input audio device overridden in getUserMedia: Expected: ' + this.attributes.audioInputId + ' Found: ' + audioTrackSettings.deviceId)
 
-				this.attributes.audioInputId = audioTrackSettings.deviceId
+				this.set('audioInputId', audioTrackSettings.deviceId)
 			}
 		}
 
@@ -311,7 +419,7 @@ MediaDevicesManager.prototype = {
 			if (videoTrackSettings && videoTrackSettings.deviceId && this.attributes.videoInputId !== videoTrackSettings.deviceId) {
 				console.debug('Input video device overridden in getUserMedia: Expected: ' + this.attributes.videoInputId + ' Found: ' + videoTrackSettings.deviceId)
 
-				this.attributes.videoInputId = videoTrackSettings.deviceId
+				this.set('videoInputId', videoTrackSettings.deviceId)
 			}
 		}
 	},

src/utils/webrtc/index.js

@@ -19,6 +19,8 @@
  *
  */
 
+import './shims/MediaStream'
+import './shims/MediaStreamTrack'
 import Axios from '@nextcloud/axios'
 import CancelableRequest from '../cancelableRequest'
 import Signaling from '../signaling'

src/utils/webrtc/models/LocalMediaModel.js

@@ -44,6 +44,7 @@ export default function LocalMediaModel() {
 	this._handleLocalStreamBound = this._handleLocalStream.bind(this)
 	this._handleLocalStreamRequestFailedRetryNoVideoBound = this._handleLocalStreamRequestFailedRetryNoVideo.bind(this)
 	this._handleLocalStreamRequestFailedBound = this._handleLocalStreamRequestFailed.bind(this)
+	this._handleLocalStreamChangedBound = this._handleLocalStreamChanged.bind(this)
 	this._handleLocalStreamStoppedBound = this._handleLocalStreamStopped.bind(this)
 	this._handleAudioOnBound = this._handleAudioOn.bind(this)
 	this._handleAudioOffBound = this._handleAudioOff.bind(this)
@@ -116,6 +117,7 @@ LocalMediaModel.prototype = {
 			this._webRtc.webrtc.off('localStream', this._handleLocalStreamBound)
 			this._webRtc.webrtc.off('localStreamRequestFailedRetryNoVideo', this._handleLocalStreamRequestFailedBound)
 			this._webRtc.webrtc.off('localStreamRequestFailed', this._handleLocalStreamRequestFailedBound)
+			this._webRtc.webrtc.off('localStreamChanged', this._handleLocalStreamChangedBound)
 			this._webRtc.webrtc.off('localStreamStopped', this._handleLocalStreamStoppedBound)
 			this._webRtc.webrtc.off('audioOn', this._handleAudioOnBound)
 			this._webRtc.webrtc.off('audioOff', this._handleAudioOffBound)
@@ -147,6 +149,7 @@ LocalMediaModel.prototype = {
 		this._webRtc.webrtc.on('localStream', this._handleLocalStreamBound)
 		this._webRtc.webrtc.on('localStreamRequestFailedRetryNoVideo', this._handleLocalStreamRequestFailedRetryNoVideoBound)
 		this._webRtc.webrtc.on('localStreamRequestFailed', this._handleLocalStreamRequestFailedBound)
+		this._webRtc.webrtc.on('localStreamChanged', this._handleLocalStreamChangedBound)
 		this._webRtc.webrtc.on('localStreamStopped', this._handleLocalStreamStoppedBound)
 		this._webRtc.webrtc.on('audioOn', this._handleAudioOnBound)
 		this._webRtc.webrtc.on('audioOff', this._handleAudioOffBound)
@@ -225,6 +228,43 @@ LocalMediaModel.prototype = {
 		}
 	},
 
+	_handleLocalStreamChanged: function(localStream) {
+		// Only a single local stream is assumed to be active at the same time.
+		this.set('localStream', localStream)
+
+		this._updateMediaAvailability(localStream)
+	},
+
+	_updateMediaAvailability: function(localStream) {
+		if (localStream && localStream.getAudioTracks().length > 0) {
+			this.set('audioAvailable', true)
+
+			if (!this.get('audioEnabled')) {
+				// Explicitly disable the audio to ensure that it will also be
+				// disabled in the other end. Otherwise the WebRTC media could
+				// be enabled.
+				this.disableAudio()
+			}
+		} else {
+			this.disableAudio()
+			this.set('audioAvailable', false)
+		}
+
+		if (localStream && localStream.getVideoTracks().length > 0) {
+			this.set('videoAvailable', true)
+
+			if (!this.get('videoEnabled')) {
+				// Explicitly disable the video to ensure that it will also be
+				// disabled in the other end. Otherwise the WebRTC media could
+				// be enabled.
+				this.disableVideo()
+			}
+		} else {
+			this.disableVideo()
+			this.set('videoAvailable', false)
+		}
+	},
+
 	_handleLocalStreamStopped: function(localStream) {
 		if (this.get('localStream') !== localStream) {
 			return

src/utils/webrtc/shims/MediaStream.js

@@ -0,0 +1,113 @@
+/**
+ *
+ * @copyright Copyright (c) 2020, Daniel Calviño Sánchez ([email protected])
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+if (window.MediaStream) {
+	const originalMediaStreamAddTrack = window.MediaStream.prototype.addTrack
+	window.MediaStream.prototype.addTrack = function(track) {
+		let addTrackEventDispatched = false
+		const testAddTrackEvent = () => {
+			addTrackEventDispatched = true
+		}
+		this.addEventListener('addtrack', testAddTrackEvent)
+
+		originalMediaStreamAddTrack.apply(this, arguments)
+
+		this.removeEventListener('addtrack', testAddTrackEvent)
+
+		if (!addTrackEventDispatched) {
+			this.dispatchEvent(new MediaStreamTrackEvent('addtrack', { track: track }))
+		}
+	}
+
+	const originalMediaStreamRemoveTrack = window.MediaStream.prototype.removeTrack
+	window.MediaStream.prototype.removeTrack = function(track) {
+		let removeTrackEventDispatched = false
+		const testRemoveTrackEvent = () => {
+			removeTrackEventDispatched = true
+		}
+		this.addEventListener('removetrack', testRemoveTrackEvent)
+
+		originalMediaStreamRemoveTrack.apply(this, arguments)
+
+		this.removeEventListener('removetrack', testRemoveTrackEvent)
+
+		if (!removeTrackEventDispatched) {
+			this.dispatchEvent(new MediaStreamTrackEvent('removetrack', { track: track }))
+		}
+	}
+
+	// Event implementations do not support advanced parameters like "options"
+	// or "useCapture".
+	const originalMediaStreamDispatchEvent = window.MediaStream.prototype.dispatchEvent
+	const originalMediaStreamAddEventListener = window.MediaStream.prototype.addEventListener
+	const originalMediaStreamRemoveEventListener = window.MediaStream.prototype.removeEventListener
+
+	window.MediaStream.prototype.dispatchEvent = function(event) {
+		if (this._listeners && this._listeners[event.type]) {
+			this._listeners[event.type].forEach(listener => {
+				listener.apply(this, [event])
+			})
+		}
+
+		return originalMediaStreamDispatchEvent.apply(this, arguments)
+	}
+
+	let isMediaStreamDispatchEventSupported
+
+	window.MediaStream.prototype.addEventListener = function(type, listener) {
+		if (isMediaStreamDispatchEventSupported === undefined) {
+			isMediaStreamDispatchEventSupported = false
+			const testDispatchEventSupportHandler = () => {
+				isMediaStreamDispatchEventSupported = true
+			}
+			originalMediaStreamAddEventListener.apply(this, ['test-dispatch-event-support', testDispatchEventSupportHandler])
+			originalMediaStreamDispatchEvent.apply(this, [new Event('test-dispatch-event-support')])
+			originalMediaStreamRemoveEventListener(this, ['test-dispatch-event-support', testDispatchEventSupportHandler])
+
+			console.debug('Is MediaStream.dispatchEvent() supported?: ', isMediaStreamDispatchEventSupported)
+		}
+
+		if (!isMediaStreamDispatchEventSupported) {
+			if (!this._listeners) {
+				this._listeners = []
+			}
+
+			if (!this._listeners.hasOwnProperty(type)) {
+				this._listeners[type] = [listener]
+			} else if (!this._listeners[type].includes(listener)) {
+				this._listeners[type].push(listener)
+			}
+		}
+
+		return originalMediaStreamAddEventListener.apply(this, arguments)
+	}
+
+	window.MediaStream.prototype.removeEventListener = function(type, listener) {
+		if (this._listeners && this._listeners[type]) {
+			const index = this._listeners[type].indexOf(listener)
+			if (index >= 0) {
+				this._listeners[type].splice(index, 1)
+			}
+		}
+
+		return originalMediaStreamRemoveEventListener.apply(this, arguments)
+	}
+}