From 8d1219ce9fee652e0d57d0afce36efbacc743e62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Thu, 16 Jul 2020 16:55:39 +0200 Subject: [PATCH 01/12] Add wrapper for MediaDevices MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MediaDevicesManager stores the id of the audio and video input devices to be used when requesting media. If no id is set then it behaves like before, that is, letting the browser decide which device to use. Signed-off-by: Daniel Calviño Sánchez --- src/utils/webrtc/MediaDevicesManager.js | 87 +++++++++++++++++++++ src/utils/webrtc/index.js | 4 + src/utils/webrtc/simplewebrtc/localmedia.js | 9 ++- 3 files changed, 97 insertions(+), 3 deletions(-) create mode 100644 src/utils/webrtc/MediaDevicesManager.js diff --git a/src/utils/webrtc/MediaDevicesManager.js b/src/utils/webrtc/MediaDevicesManager.js new file mode 100644 index 00000000000..1f0646b4ccb --- /dev/null +++ b/src/utils/webrtc/MediaDevicesManager.js @@ -0,0 +1,87 @@ +/** + * + * @copyright Copyright (c) 2020, Daniel Calviño Sánchez (danxuliu@gmail.com) + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +/** + * Wrapper for MediaDevices to simplify its use. + * + * "attributes.audioInputId" and "attributes.videoInputId" define the devices + * that will be used when calling "getUserMedia(constraints)". + */ +export default function MediaDevicesManager() { + this.attributes = { + audioInputId: undefined, + videoInputId: undefined, + } +} +MediaDevicesManager.prototype = { + + /** + * Returns whether getting user media and enumerating media devices is + * supported or not. + * + * Note that even if false is returned the MediaDevices interface could be + * technically supported by the browser but not available due to the page + * being loaded in an insecure context. + * + * @returns {boolean} true if MediaDevices interface is supported, false + * otherwise. + */ + isSupported: function() { + return navigator && navigator.mediaDevices && navigator.mediaDevices.getUserMedia && navigator.mediaDevices.enumerateDevices + }, + + /** + * Wrapper for MediaDevices.getUserMedia to use the selected audio and video + * input devices. + * + * The selected audio and video input devices are used only if the + * constraints do not specify a device already. Otherwise the devices in the + * constraints are respected. + * + * @param {MediaStreamConstraints} constraints the constraints specifying + * the media to request + * @returns {Promise} resolved with a MediaStream object when successful, or + * rejected with a DOMException in case of error + */ + getUserMedia: function(constraints) { + if (!this.isSupported()) { + return new Promise((resolve, reject) => { + reject(new DOMException('MediaDevicesManager is not supported', 'NotSupportedError')) + }) + } + + if (constraints.audio && !constraints.audio.deviceId && this.attributes.audioInputId) { + if (!(constraints.audio instanceof Object)) { + constraints.audio = {} + } + constraints.audio.deviceId = this.attributes.audioInputId + } + + if (constraints.video && !constraints.video.deviceId && this.attributes.videoInputId) { + if (!(constraints.video instanceof Object)) { + constraints.video = {} + } + constraints.video.deviceId = this.attributes.videoInputId + } + + return navigator.mediaDevices.getUserMedia(constraints) + }, +} diff --git a/src/utils/webrtc/index.js b/src/utils/webrtc/index.js index 3d6300cf78d..4f5aa5a4907 100644 --- a/src/utils/webrtc/index.js +++ b/src/utils/webrtc/index.js @@ -27,6 +27,7 @@ import CallAnalyzer from './analyzers/CallAnalyzer' import CallParticipantCollection from './models/CallParticipantCollection' import LocalCallParticipantModel from './models/LocalCallParticipantModel' import LocalMediaModel from './models/LocalMediaModel' +import MediaDevicesManager from './MediaDevicesManager' import SentVideoQualityThrottler from './SentVideoQualityThrottler' import { PARTICIPANT } from '../../constants' import { fetchSignalingSettings } from '../../services/signalingService' @@ -35,6 +36,7 @@ let webRtc = null const callParticipantCollection = new CallParticipantCollection() const localCallParticipantModel = new LocalCallParticipantModel() const localMediaModel = new LocalMediaModel() +const mediaDevicesManager = new MediaDevicesManager() let callAnalyzer = null let sentVideoQualityThrottler = null @@ -225,6 +227,8 @@ export { localCallParticipantModel, localMediaModel, + mediaDevicesManager, + callAnalyzer, signalingJoinConversation, diff --git a/src/utils/webrtc/simplewebrtc/localmedia.js b/src/utils/webrtc/simplewebrtc/localmedia.js index b49a4d744c7..61b0955d22e 100644 --- a/src/utils/webrtc/simplewebrtc/localmedia.js +++ b/src/utils/webrtc/simplewebrtc/localmedia.js @@ -5,6 +5,9 @@ const hark = require('hark') const getScreenMedia = require('./getscreenmedia') const WildEmitter = require('wildemitter') const mockconsole = require('mockconsole') +// Only mediaDevicesManager is used, but it can not be assigned here due to not +// being initialized yet. +const webrtcIndex = require('../index.js') function isAllTracksEnded(stream) { let isAllTracksEnded = true @@ -43,7 +46,7 @@ function LocalMedia(opts) { this._audioMonitorStreams = [] this.localScreens = [] - if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) { + if (!webrtcIndex.mediaDevicesManager.isSupported()) { this._logerror('Your browser does not support local media capture.') } @@ -86,7 +89,7 @@ LocalMedia.prototype.start = function(mediaConstraints, cb, context) { const self = this const constraints = mediaConstraints || this.config.media - if (!navigator || !navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) { + if (!webrtcIndex.mediaDevicesManager.isSupported()) { const error = new Error('MediaStreamError') error.name = 'NotSupportedError' @@ -99,7 +102,7 @@ LocalMedia.prototype.start = function(mediaConstraints, cb, context) { this.emit('localStreamRequested', constraints, context) - navigator.mediaDevices.getUserMedia(constraints).then(function(stream) { + webrtcIndex.mediaDevicesManager.getUserMedia(constraints).then(function(stream) { // Although the promise should be resolved only if all the constraints // are met Edge resolves it if both audio and video are requested but // only audio is available. From e59629b55dca4965a68284165495649ea3f04cd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Thu, 16 Jul 2020 17:00:45 +0200 Subject: [PATCH 02/12] Keep track of devices in MediaDevicesManager MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Daniel Calviño Sánchez --- src/utils/webrtc/MediaDevicesManager.js | 130 +++++++++++++++++++++++- 1 file changed, 129 insertions(+), 1 deletion(-) diff --git a/src/utils/webrtc/MediaDevicesManager.js b/src/utils/webrtc/MediaDevicesManager.js index 1f0646b4ccb..8f67d016893 100644 --- a/src/utils/webrtc/MediaDevicesManager.js +++ b/src/utils/webrtc/MediaDevicesManager.js @@ -22,14 +22,44 @@ /** * Wrapper for MediaDevices to simplify its use. * + * The MediaDevicesManager keeps an updated list of devices that can be accessed + * from "attributes.devices". Clients of this class must call + * "enableDeviceEvents()" to start keeping track of the devices, and + * "disableDeviceEvents()" once it is no longer needed. Eventually there must be + * one call to "disableDeviceEvents()" for each call to "enableDeviceEvents()", + * but several clients can be active at the same time. + * + * Each element of "attributes.devices" is an object with the following fields: + * - deviceId: the unique identifier for the device + * - groupId: two or more devices have the same groupId if they belong to the + * same physical device + * - kind: either "audioinput", "videoinput" or "audiooutput" + * - label: a human readable identifier for the device + * + * Note that the list may not contain some kind of devices due to browser + * limitations (for example, currently Firefox does not list "audiooutput" + * devices). + * + * The label may not be available if persistent media permissions have not been + * granted and a MediaStream has not been active. + * * "attributes.audioInputId" and "attributes.videoInputId" define the devices * that will be used when calling "getUserMedia(constraints)". + * + * The selected devices will be automatically cleared if they are no longer + * available. */ export default function MediaDevicesManager() { this.attributes = { + devices: [], + audioInputId: undefined, videoInputId: undefined, } + + this._enabledCount = 0 + + this._updateDevicesBound = this._updateDevices.bind(this) } MediaDevicesManager.prototype = { @@ -48,6 +78,90 @@ MediaDevicesManager.prototype = { return navigator && navigator.mediaDevices && navigator.mediaDevices.getUserMedia && navigator.mediaDevices.enumerateDevices }, + enableDeviceEvents: function() { + if (!this.isSupported()) { + return + } + + this._enabledCount++ + + this._updateDevices() + + navigator.mediaDevices.addEventListener('devicechange', this._updateDevicesBound) + }, + + disableDeviceEvents: function() { + if (!this.isSupported()) { + return + } + + this._enabledCount-- + + if (!this._enabledCount) { + navigator.mediaDevices.removeEventListener('devicechange', this._updateDevicesBound) + } + }, + + _updateDevices: function() { + navigator.mediaDevices.enumerateDevices().then(devices => { + const removedDevices = this.attributes.devices.filter(oldDevice => !devices.find(device => oldDevice.deviceId === device.deviceId && oldDevice.kind === device.kind)) + const updatedDevices = devices.filter(device => this.attributes.devices.find(oldDevice => device.deviceId === oldDevice.deviceId && device.kind === oldDevice.kind)) + const addedDevices = devices.filter(device => !this.attributes.devices.find(oldDevice => device.deviceId === oldDevice.deviceId && device.kind === oldDevice.kind)) + + removedDevices.forEach(removedDevice => { + this._removeDevice(removedDevice) + }) + updatedDevices.forEach(updatedDevice => { + this._updateDevice(updatedDevice) + }) + addedDevices.forEach(addedDevice => { + this._addDevice(addedDevice) + }) + }).catch(function(error) { + console.error('Could not update known media devices: ' + error.name + ': ' + error.message) + }) + }, + + _removeDevice: function(removedDevice) { + if (removedDevice.kind === 'audioinput' && this.attributes.audioInputId === removedDevice.deviceId) { + this.attributes.audioInputId = undefined + } else if (removedDevice.kind === 'videoinput' && this.attributes.videoInputId === removedDevice.deviceId) { + this.attributes.videoInputId = undefined + } + + const removedDeviceIndex = this.attributes.devices.findIndex(oldDevice => oldDevice.deviceId === removedDevice.deviceId && oldDevice.kind === removedDevice.kind) + if (removedDeviceIndex >= 0) { + this.attributes.devices.splice(removedDeviceIndex, 1) + } + }, + + _updateDevice: function(updatedDevice) { + const oldDevice = this.attributes.devices.find(oldDevice => oldDevice.deviceId === updatedDevice.deviceId && oldDevice.kind === updatedDevice.kind) + + // Only update the label if it has a value, as it may have been + // removed if there is currently no active stream. + if (updatedDevice.label) { + oldDevice.label = updatedDevice.label + } + + // These should not have changed, but just in case + oldDevice.groupId = updatedDevice.groupId + oldDevice.kind = updatedDevice.kind + }, + + _addDevice: function(addedDevice) { + // Copy the device to add, as its properties are read only and + // thus they can not be updated later. + addedDevice = { + deviceId: addedDevice.deviceId, + groupId: addedDevice.groupId, + kind: addedDevice.kind, + label: addedDevice.label, + } + + this.attributes.devices.push(addedDevice) + }, + /** * Wrapper for MediaDevices.getUserMedia to use the selected audio and video * input devices. @@ -82,6 +196,20 @@ MediaDevicesManager.prototype = { constraints.video.deviceId = this.attributes.videoInputId } - return navigator.mediaDevices.getUserMedia(constraints) + return navigator.mediaDevices.getUserMedia(constraints).then(stream => { + // The list of devices is always updated when a stream is started as + // that is the only time at which the full device information is + // guaranteed to be available. + this._updateDevices() + + return stream + }).catch(error => { + // The list of devices is also updated in case of failure, as even + // if getting the stream failed the permissions may have been + // permanently granted. + this._updateDevices() + + throw error + }) }, } From 5cc70315eea0ee7feccbf2cf63361c1a3b6c40fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Thu, 16 Jul 2020 17:08:00 +0200 Subject: [PATCH 03/12] Add fallback label to devices in MediaDevicesManager MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The device label returned by "MediaDevices.enumerateDevices()" is empty if permanent media permissions have not been granted and there is no MediaStream currently active. Therefore a fallback label is generated to help the user differentiate between devices. The labels are persistent during the session, so if a device is connected, disconnected and then connected again some time later after other devices have been connected too it will retain its original fallback label. Signed-off-by: Daniel Calviño Sánchez --- src/utils/webrtc/MediaDevicesManager.js | 29 ++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/src/utils/webrtc/MediaDevicesManager.js b/src/utils/webrtc/MediaDevicesManager.js index 8f67d016893..d74128e6e30 100644 --- a/src/utils/webrtc/MediaDevicesManager.js +++ b/src/utils/webrtc/MediaDevicesManager.js @@ -35,13 +35,15 @@ * same physical device * - kind: either "audioinput", "videoinput" or "audiooutput" * - label: a human readable identifier for the device + * - fallbackLabel: a generated label if the actual label is empty * * Note that the list may not contain some kind of devices due to browser * limitations (for example, currently Firefox does not list "audiooutput" * devices). * * The label may not be available if persistent media permissions have not been - * granted and a MediaStream has not been active. + * granted and a MediaStream has not been active. In those cases the fallback + * label can be used instead. * * "attributes.audioInputId" and "attributes.videoInputId" define the devices * that will be used when calling "getUserMedia(constraints)". @@ -59,6 +61,8 @@ export default function MediaDevicesManager() { this._enabledCount = 0 + this._knownDevices = {} + this._updateDevicesBound = this._updateDevices.bind(this) } MediaDevicesManager.prototype = { @@ -159,6 +163,29 @@ MediaDevicesManager.prototype = { label: addedDevice.label, } + const knownDevice = this._knownDevices[addedDevice.kind + '-' + addedDevice.deviceId] + if (knownDevice) { + addedDevice.fallbackLabel = knownDevice.fallbackLabel + // If the added device has a label keep it; otherwise use + // the previously known one, if any. + addedDevice.label = addedDevice.label ? addedDevice.label : knownDevice.label + } else { + // Generate a fallback label to be used when the actual label is + // not available. + if (addedDevice.deviceId === 'default') { + addedDevice.fallbackLabel = t('spreed', 'Default') + } else if (addedDevice.kind === 'audioinput') { + addedDevice.fallbackLabel = t('spreed', 'Microphone {number}', { number: Object.values(this._knownDevices).filter(device => device.kind === 'audioinput').length + 1 }) + } else if (addedDevice.kind === 'videoinput') { + addedDevice.fallbackLabel = t('spreed', 'Camera {number}', { number: Object.values(this._knownDevices).filter(device => device.kind === 'videoinput').length + 1 }) + } else if (addedDevice.kind === 'audiooutput') { + addedDevice.fallbackLabel = t('spreed', 'Speaker {number}', { number: Object.values(this._knownDevices).filter(device => device.kind === 'audioutput').length + 1 }) + } + } + + // Always refresh the known device with the latest values. + this._knownDevices[addedDevice.kind + '-' + addedDevice.deviceId] = addedDevice + this.attributes.devices.push(addedDevice) }, From 8832daf8c9fb838dabc4b140a647fe5d50d289b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Thu, 16 Jul 2020 17:19:44 +0200 Subject: [PATCH 04/12] Add components to select and preview media devices MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MediaDevicesSelector is a dropdown to select a media device of a specific kind. MediaDevicesPreview has selectors for audio and video inputs that modify the devices to be used in MediaDevicesManager. Additionally it also shows a preview of the selected devices with a volume bar in the case of audio or a video element in the case of video. Signed-off-by: Daniel Calviño Sánchez --- src/components/MediaDevicesPreview.vue | 407 ++++++++++++++++++++++++ src/components/MediaDevicesSelector.vue | 166 ++++++++++ 2 files changed, 573 insertions(+) create mode 100644 src/components/MediaDevicesPreview.vue create mode 100644 src/components/MediaDevicesSelector.vue diff --git a/src/components/MediaDevicesPreview.vue b/src/components/MediaDevicesPreview.vue new file mode 100644 index 00000000000..5cd687f4c9d --- /dev/null +++ b/src/components/MediaDevicesPreview.vue @@ -0,0 +1,407 @@ + + + + + + + diff --git a/src/components/MediaDevicesSelector.vue b/src/components/MediaDevicesSelector.vue new file mode 100644 index 00000000000..e043695b4d5 --- /dev/null +++ b/src/components/MediaDevicesSelector.vue @@ -0,0 +1,166 @@ + + + + + + + From 23b9267d24f5b64f3d58a5b409cc8685fa78fa95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Thu, 16 Jul 2020 17:41:27 +0200 Subject: [PATCH 05/12] Add fallback devices to DeviceMediaManager MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When there is no selected device (either because the selected one was removed or because there were no devices before) but there are other devices now a fallback device is automatically selected. Signed-off-by: Daniel Calviño Sánchez --- src/utils/webrtc/MediaDevicesManager.js | 49 +++++++++++++++++++++---- 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/src/utils/webrtc/MediaDevicesManager.js b/src/utils/webrtc/MediaDevicesManager.js index d74128e6e30..3c005375198 100644 --- a/src/utils/webrtc/MediaDevicesManager.js +++ b/src/utils/webrtc/MediaDevicesManager.js @@ -49,7 +49,9 @@ * that will be used when calling "getUserMedia(constraints)". * * The selected devices will be automatically cleared if they are no longer - * available. + * available. When no device of certain kind is selected and there are other + * devices of that kind the selected device will fall back to the first one + * found, or to the one with the "default" id (if any). */ export default function MediaDevicesManager() { this.attributes = { @@ -63,6 +65,9 @@ export default function MediaDevicesManager() { this._knownDevices = {} + this._fallbackAudioInputId = undefined + this._fallbackVideoInputId = undefined + this._updateDevicesBound = this._updateDevices.bind(this) } MediaDevicesManager.prototype = { @@ -127,16 +132,28 @@ MediaDevicesManager.prototype = { }, _removeDevice: function(removedDevice) { - if (removedDevice.kind === 'audioinput' && this.attributes.audioInputId === removedDevice.deviceId) { - this.attributes.audioInputId = undefined - } else if (removedDevice.kind === 'videoinput' && this.attributes.videoInputId === removedDevice.deviceId) { - this.attributes.videoInputId = undefined - } - const removedDeviceIndex = this.attributes.devices.findIndex(oldDevice => oldDevice.deviceId === removedDevice.deviceId && oldDevice.kind === removedDevice.kind) if (removedDeviceIndex >= 0) { this.attributes.devices.splice(removedDeviceIndex, 1) } + + if (removedDevice.kind === 'audioinput') { + if (this._fallbackAudioInputId === removedDevice.deviceId) { + const firstAudioInputDevice = this.attributes.devices.find(device => device.kind === 'audioinput') + this._fallbackAudioInputId = firstAudioInputDevice ? firstAudioInputDevice.deviceId : undefined + } + if (this.attributes.audioInputId === removedDevice.deviceId) { + this.attributes.audioInputId = this._fallbackAudioInputId + } + } else if (removedDevice.kind === 'videoinput') { + if (this._fallbackVideoInputId === removedDevice.deviceId) { + const firstVideoInputDevice = this.attributes.devices.find(device => device.kind === 'videoinput') + this._fallbackVideoInputId = firstVideoInputDevice ? firstVideoInputDevice.deviceId : undefined + } + if (this.attributes.videoInputId === removedDevice.deviceId) { + this.attributes.videoInputId = this._fallbackVideoInputId + } + } }, _updateDevice: function(updatedDevice) { @@ -186,6 +203,24 @@ MediaDevicesManager.prototype = { // Always refresh the known device with the latest values. this._knownDevices[addedDevice.kind + '-' + addedDevice.deviceId] = addedDevice + // Set first available device as fallback, and override any + // fallback previously set if the default device is added. + if (addedDevice.kind === 'audioinput') { + if (!this._fallbackAudioInputId || addedDevice.deviceId === 'default') { + this._fallbackAudioInputId = addedDevice.deviceId + } + if (this.attributes.audioInputId === undefined) { + this.attributes.audioInputId = this._fallbackAudioInputId + } + } else if (addedDevice.kind === 'videoinput') { + if (!this._fallbackVideoInputId || addedDevice.deviceId === 'default') { + this._fallbackVideoInputId = addedDevice.deviceId + } + if (this.attributes.videoInputId === undefined) { + this.attributes.videoInputId = this._fallbackVideoInputId + } + } + this.attributes.devices.push(addedDevice) }, From fd108bb404eda6c322467d44b2686d9522f43e37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Thu, 16 Jul 2020 17:44:45 +0200 Subject: [PATCH 06/12] Make possible to explicitly disable devices of certain kind MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now it is possible to disable audio and video devices by setting the selected id to null. Fallback devices will still be used when the selected device is undefined, but not when it is explicitly set to null. Signed-off-by: Daniel Calviño Sánchez --- src/components/MediaDevicesSelector.vue | 11 ++++++++-- src/utils/webrtc/MediaDevicesManager.js | 28 +++++++++++++++++-------- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/src/components/MediaDevicesSelector.vue b/src/components/MediaDevicesSelector.vue index e043695b4d5..843716b77d7 100644 --- a/src/components/MediaDevicesSelector.vue +++ b/src/components/MediaDevicesSelector.vue @@ -88,7 +88,7 @@ export default { }, deviceOptionsAvailable() { - return this.deviceOptions.length > 0 + return this.deviceOptions.length > 1 }, deviceSelectorPlaceholder() { @@ -120,12 +120,19 @@ export default { }, deviceOptions() { - return this.devices.filter(device => device.kind === this.kind).map(device => { + const options = this.devices.filter(device => device.kind === this.kind).map(device => { return { id: device.deviceId, label: device.label ? device.label : device.fallbackLabel, } }) + + options.push({ + id: null, + label: t('spreed', 'None'), + }) + + return options }, deviceSelectedOptionFromDeviceId() { diff --git a/src/utils/webrtc/MediaDevicesManager.js b/src/utils/webrtc/MediaDevicesManager.js index 3c005375198..b06457e6e48 100644 --- a/src/utils/webrtc/MediaDevicesManager.js +++ b/src/utils/webrtc/MediaDevicesManager.js @@ -51,7 +51,9 @@ * The selected devices will be automatically cleared if they are no longer * available. When no device of certain kind is selected and there are other * devices of that kind the selected device will fall back to the first one - * found, or to the one with the "default" id (if any). + * found, or to the one with the "default" id (if any). It is possible to + * explicitly disable devices of certain kind by setting xxxInputId to "null" + * (in that case the fallback devices will not be taken into account). */ export default function MediaDevicesManager() { this.attributes = { @@ -244,18 +246,26 @@ MediaDevicesManager.prototype = { }) } - if (constraints.audio && !constraints.audio.deviceId && this.attributes.audioInputId) { - if (!(constraints.audio instanceof Object)) { - constraints.audio = {} + if (constraints.audio && !constraints.audio.deviceId) { + if (this.attributes.audioInputId) { + if (!(constraints.audio instanceof Object)) { + constraints.audio = {} + } + constraints.audio.deviceId = this.attributes.audioInputId + } else if (this.attributes.audioInputId === null) { + constraints.audio = false } - constraints.audio.deviceId = this.attributes.audioInputId } - if (constraints.video && !constraints.video.deviceId && this.attributes.videoInputId) { - if (!(constraints.video instanceof Object)) { - constraints.video = {} + if (constraints.video && !constraints.video.deviceId) { + if (this.attributes.videoInputId) { + if (!(constraints.video instanceof Object)) { + constraints.video = {} + } + constraints.video.deviceId = this.attributes.videoInputId + } else if (this.attributes.videoInputId === null) { + constraints.video = false } - constraints.video.deviceId = this.attributes.videoInputId } return navigator.mediaDevices.getUserMedia(constraints).then(stream => { From 1cf15e4e414ead24721c7abc2714223f1b54a236 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Thu, 16 Jul 2020 17:47:18 +0200 Subject: [PATCH 07/12] Add "Preview" tab to the right sidebar MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The preview tab makes possible to select the audio and video devices to be used in a call, as well as showing a preview of them (so the user now can check what is visible in the camera before joining a call). In order to show the preview of the devices a media stream has to be requested, which will ask the user for permissions (unless they have been granted permanently). To prevent permissions requests as soon as the conversation is opened the preview tab is not shown as the first tab, and to limit the use of the devices only to the scrictly necessary the previews are enabled only when the preview tab is the active one. Currently changing the devices during a call has no effect, so the device selectors (as well as the previews themselves, as they will have the same content already shown in the call view) are disabled during calls. The tab itself is not removed due to ordering issues when removing and adding it back. Signed-off-by: Daniel Calviño Sánchez --- src/components/RightSidebar/RightSidebar.vue | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/components/RightSidebar/RightSidebar.vue b/src/components/RightSidebar/RightSidebar.vue index 2ca6af13eb7..6d694bdcb37 100644 --- a/src/components/RightSidebar/RightSidebar.vue +++ b/src/components/RightSidebar/RightSidebar.vue @@ -27,6 +27,7 @@ :title="title" :starred="isFavorited" :title-editable="canModerate && isRenamingConversation" + @update:active="handleUpdateActive" @update:starred="onFavoriteChange" @update:title="handleUpdateTitle" @submit-title="handleSubmitTitle" @@ -66,6 +67,13 @@ icon="icon-settings"> + + + @@ -73,6 +81,7 @@ import AppSidebar from '@nextcloud/vue/dist/Components/AppSidebar' import AppSidebarTab from '@nextcloud/vue/dist/Components/AppSidebarTab' import ChatView from '../ChatView' +import MediaDevicesPreview from '../MediaDevicesPreview' import { CollectionList } from 'nextcloud-vue-collections' import BrowserStorage from '../../services/BrowserStorage' import { CONVERSATION, WEBINAR, PARTICIPANT } from '../../constants' @@ -92,6 +101,7 @@ export default { AppSidebarTab, ChatView, CollectionList, + MediaDevicesPreview, ParticipantsTab, SetGuestUsername, }, @@ -109,6 +119,7 @@ export default { data() { return { + activeTab: null, contactsLoading: false, // The conversation name (while editing) conversationName: '', @@ -215,6 +226,10 @@ export default { this.conversation.isFavorite = !this.conversation.isFavorite }, + handleUpdateActive(active) { + this.activeTab = active + }, + /** * Updates the conversationName value while editing the conversation's title. * @param {string} title the conversation title emitted by the AppSidevar vue From 50d772d5fc49cf229d4caad8a51444151d02f234 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Tue, 4 Aug 2020 11:44:51 +0200 Subject: [PATCH 08/12] Show also an error message instead of just an error icon MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Daniel Calviño Sánchez --- src/components/MediaDevicesPreview.vue | 79 +++++++++++++++++++++++--- 1 file changed, 72 insertions(+), 7 deletions(-) diff --git a/src/components/MediaDevicesPreview.vue b/src/components/MediaDevicesPreview.vue index 5cd687f4c9d..4bc9b1bb404 100644 --- a/src/components/MediaDevicesPreview.vue +++ b/src/components/MediaDevicesPreview.vue @@ -36,6 +36,9 @@ class="icon icon-audio" />
+

+ {{ audioStreamErrorMessage }} +

@@ -63,6 +66,9 @@ class="icon icon-video" />
+

+ {{ videoStreamErrorMessage }} +

@@ -99,9 +105,9 @@ export default { mounted: false, mediaDevicesManager: mediaDevicesManager, audioStream: null, - audioStreamError: false, + audioStreamError: null, videoStream: null, - videoStreamError: false, + videoStreamError: null, hark: null, currentVolume: -100, volumeThreshold: -100, @@ -139,6 +145,58 @@ export default { return this.videoInputId && this.videoStream }, + audioStreamErrorMessage() { + if (!this.audioStreamError) { + return null + } + + if (this.audioStreamError.name === 'NotSupportedError' && !window.RTCPeerConnection) { + return t('spreed', 'Calls are not supported in your browser') + } + + // In newer browser versions MediaDevicesManager is not supported in + // insecure contexts; in older browser versions it is, but getting + // the user media fails with "NotAllowedError". + const isInsecureContext = 'isSecureContext' in window && !window.isSecureContext + const isInsecureContextAccordingToErrorMessage = this.audioStreamError.message && this.audioStreamError.message.indexOf('Only secure origins') !== -1 + if ((this.audioStreamError.name === 'NotSupportedError' && isInsecureContext) + || (this.audioStreamError.name === 'NotAllowedError' && isInsecureContextAccordingToErrorMessage)) { + return t('spreed', 'Access to microphone is only possible with HTTPS') + } + + if (this.audioStreamError.name === 'NotAllowedError') { + return t('spreed', 'Access to microphone was denied') + } + + return t('spreed', 'Error while accessing microphone') + }, + + videoStreamErrorMessage() { + if (!this.videoStreamError) { + return null + } + + if (this.videoStreamError.name === 'NotSupportedError' && !window.RTCPeerConnection) { + return t('spreed', 'Calls are not supported in your browser') + } + + // In newer browser versions MediaDevicesManager is not supported in + // insecure contexts; in older browser versions it is, but getting + // the user media fails with "NotAllowedError". + const isInsecureContext = 'isSecureContext' in window && !window.isSecureContext + const isInsecureContextAccordingToErrorMessage = this.videoStreamError.message && this.videoStreamError.message.indexOf('Only secure origins') !== -1 + if ((this.videoStreamError.name === 'NotSupportedError' && isInsecureContext) + || (this.videoStreamError.name === 'NotAllowedError' && isInsecureContextAccordingToErrorMessage)) { + return t('spreed', 'Access to camera is only possible with HTTPS') + } + + if (this.videoStreamError.name === 'NotAllowedError') { + return t('spreed', 'Access to camera was denied') + } + + return t('spreed', 'Error while accessing camera') + }, + currentVolumeIndicatorHeight() { // refs can not be accessed on the initial render, only after the // component has been mounted. @@ -233,7 +291,7 @@ export default { // https://bugzilla.mozilla.org/show_bug.cgi?id=1468700 this.stopAudioStream() - this.audioStreamError = false + this.audioStreamError = null if (!this.audioInputId) { return @@ -245,7 +303,7 @@ export default { }) .catch(error => { console.error('Error getting audio stream: ' + error.name + ': ' + error.message) - this.audioStreamError = true + this.audioStreamError = error this.setAudioStream(null) }) }, @@ -259,7 +317,7 @@ export default { // the audio ones (see "updateAudioStream"). this.stopVideoStream() - this.videoStreamError = false + this.videoStreamError = null if (!this.videoInputId) { return @@ -271,7 +329,7 @@ export default { }) .catch(error => { console.error('Error getting video stream: ' + error.name + ': ' + error.message) - this.videoStreamError = true + this.videoStreamError = error this.setVideoStream(null) }) }, @@ -356,9 +414,16 @@ export default { width: 64px; height: 64px; opacity: 0.4; + + margin-left: auto; + margin-right: auto; } } +.preview-not-available p { + margin-bottom: 16px; +} + .preview-audio { .preview-not-available .icon { margin-top: 16px; @@ -396,7 +461,7 @@ export default { .preview-video { .preview-not-available .icon { margin-top: 64px; - margin-bottom: 64px; + margin-bottom: 16px; } video { From a116250409096f9fd1e9bb8aa1dd17d970208560 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Tue, 4 Aug 2020 12:24:25 +0200 Subject: [PATCH 09/12] Do not update streams again while waiting for a previous one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When an audio or video device is selected a stream from that device is requested to show its preview. The stream is resolved asynchronously, so while that happens the user could select a different device, which in turn will request another stream. As only a single stream for each type is expected to be active in the previews this could lead to streams being opened but never closed. Now this is enforced by preventing further stream requests while a previous one has not been completed yet. If several stream updates are triggered while waiting for a previous one once that previous one is finished a new stream will be requested for the last selected device. Signed-off-by: Daniel Calviño Sánchez --- src/components/MediaDevicesPreview.vue | 34 ++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/components/MediaDevicesPreview.vue b/src/components/MediaDevicesPreview.vue index 4bc9b1bb404..6c2137bd6a6 100644 --- a/src/components/MediaDevicesPreview.vue +++ b/src/components/MediaDevicesPreview.vue @@ -104,6 +104,8 @@ export default { return { mounted: false, mediaDevicesManager: mediaDevicesManager, + pendingGetUserMediaAudioCount: 0, + pendingGetUserMediaVideoCount: 0, audioStream: null, audioStreamError: null, videoStream: null, @@ -284,6 +286,12 @@ export default { return } + if (this.pendingGetUserMediaAudioCount) { + this.pendingGetUserMediaAudioCount++ + + return + } + // When the audio input device changes the previous stream must be // stopped before a new one is requested, as for example currently // Firefox does not support having two different audio input devices @@ -297,6 +305,8 @@ export default { return } + this.pendingGetUserMediaAudioCount = 1 + this.mediaDevicesManager.getUserMedia({ audio: true }) .then(stream => { this.setAudioStream(stream) @@ -305,6 +315,14 @@ export default { console.error('Error getting audio stream: ' + error.name + ': ' + error.message) this.audioStreamError = error this.setAudioStream(null) + }).finally(() => { + const updateAudioStreamAgain = this.pendingGetUserMediaAudioCount > 1 + + this.pendingGetUserMediaAudioCount = 0 + + if (updateAudioStreamAgain) { + this.updateAudioStream() + } }) }, @@ -313,6 +331,12 @@ export default { return } + if (this.pendingGetUserMediaVideoCount) { + this.pendingGetUserMediaVideoCount++ + + return + } + // Video stream is stopped too to avoid potential issues similar to // the audio ones (see "updateAudioStream"). this.stopVideoStream() @@ -323,6 +347,8 @@ export default { return } + this.pendingGetUserMediaVideoCount = 1 + this.mediaDevicesManager.getUserMedia({ video: true }) .then(stream => { this.setVideoStream(stream) @@ -331,6 +357,14 @@ export default { console.error('Error getting video stream: ' + error.name + ': ' + error.message) this.videoStreamError = error this.setVideoStream(null) + }).finally(() => { + const updateVideoStreamAgain = this.pendingGetUserMediaVideoCount > 1 + + this.pendingGetUserMediaVideoCount = 0 + + if (updateVideoStreamAgain) { + this.updateVideoStream() + } }) }, From 51a9aededb763d4f67b90ddafc3482df8071bcbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Tue, 4 Aug 2020 12:48:32 +0200 Subject: [PATCH 10/12] Do not request a stream again for the active device MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a preview stream is updated there is no need to request the stream again if the current stream comes already from the selected device. Signed-off-by: Daniel Calviño Sánchez --- src/components/MediaDevicesPreview.vue | 34 ++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/components/MediaDevicesPreview.vue b/src/components/MediaDevicesPreview.vue index 6c2137bd6a6..7fe42c83749 100644 --- a/src/components/MediaDevicesPreview.vue +++ b/src/components/MediaDevicesPreview.vue @@ -139,6 +139,32 @@ export default { }, }, + audioStreamInputId() { + if (!this.audioStream) { + return null + } + + const audioTracks = this.audioStream.getAudioTracks() + if (audioTracks.length < 1) { + return null + } + + return audioTracks[0].getSettings().deviceId + }, + + videoStreamInputId() { + if (!this.videoStream) { + return null + } + + const videoTracks = this.videoStream.getVideoTracks() + if (videoTracks.length < 1) { + return null + } + + return videoTracks[0].getSettings().deviceId + }, + audioPreviewAvailable() { return this.audioInputId && this.audioStream }, @@ -286,6 +312,10 @@ export default { return } + if (this.audioStreamInputId && this.audioStreamInputId === this.audioInputId) { + return + } + if (this.pendingGetUserMediaAudioCount) { this.pendingGetUserMediaAudioCount++ @@ -331,6 +361,10 @@ export default { return } + if (this.videoStreamInputId && this.videoStreamInputId === this.videoInputId) { + return + } + if (this.pendingGetUserMediaVideoCount) { this.pendingGetUserMediaVideoCount++ From ab10ed58b40c14d3ba6e27b982bb81e1204b246d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Tue, 4 Aug 2020 13:28:06 +0200 Subject: [PATCH 11/12] Update selected devices from the getUserMedia results MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In Firefox the dialog to grant media permissions allows the user to change the device to use. In that case the device selected in the dialog will be the one used, no matter which one was originally requested. Now this is taken into account and the selected device is updated to reflect the one selected by the user in the dialog. Signed-off-by: Daniel Calviño Sánchez --- src/utils/webrtc/MediaDevicesManager.js | 27 +++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/utils/webrtc/MediaDevicesManager.js b/src/utils/webrtc/MediaDevicesManager.js index b06457e6e48..53cd4b236a5 100644 --- a/src/utils/webrtc/MediaDevicesManager.js +++ b/src/utils/webrtc/MediaDevicesManager.js @@ -269,6 +269,11 @@ MediaDevicesManager.prototype = { } return navigator.mediaDevices.getUserMedia(constraints).then(stream => { + // In Firefox the dialog to grant media permissions allows the user + // to change the device to use, overriding the device that was + // originally requested. + this._updateSelectedDevicesFromGetUserMediaResult(stream) + // The list of devices is always updated when a stream is started as // that is the only time at which the full device information is // guaranteed to be available. @@ -284,4 +289,26 @@ MediaDevicesManager.prototype = { throw error }) }, + + _updateSelectedDevicesFromGetUserMediaResult: function(stream) { + if (this.attributes.audioInputId) { + const audioTracks = stream.getAudioTracks() + const audioTrackSettings = audioTracks.length > 0 ? audioTracks[0].getSettings() : null + if (audioTrackSettings && audioTrackSettings.deviceId && this.attributes.audioInputId !== audioTrackSettings.deviceId) { + console.debug('Input audio device overridden in getUserMedia: Expected: ' + this.attributes.audioInputId + ' Found: ' + audioTrackSettings.deviceId) + + this.attributes.audioInputId = audioTrackSettings.deviceId + } + } + + if (this.attributes.videoInputId) { + const videoTracks = stream.getVideoTracks() + const videoTrackSettings = videoTracks.length > 0 ? videoTracks[0].getSettings() : null + if (videoTrackSettings && videoTrackSettings.deviceId && this.attributes.videoInputId !== videoTrackSettings.deviceId) { + console.debug('Input video device overridden in getUserMedia: Expected: ' + this.attributes.videoInputId + ' Found: ' + videoTrackSettings.deviceId) + + this.attributes.videoInputId = videoTrackSettings.deviceId + } + } + }, } From ec7fa54fafb326c111ce7a8c61a6d8e3b46c4093 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Wed, 5 Aug 2020 03:55:12 +0200 Subject: [PATCH 12/12] Handle empty "deviceId" when media permissions are not granted MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In latest MediaDevices spec if permanent media permissions have not been granted and there is no active stream "enumerateDevices" returns at most one device of each kind, and all of them with empty attributes (including the deviceId) except for the kind. This is already partially implemented by Chromium, so devices with an empty "deviceId" need to be taken into account. Signed-off-by: Daniel Calviño Sánchez --- src/components/MediaDevicesPreview.vue | 4 ++-- src/utils/webrtc/MediaDevicesManager.js | 18 +++++++++++------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/components/MediaDevicesPreview.vue b/src/components/MediaDevicesPreview.vue index 7fe42c83749..ed82767c6cc 100644 --- a/src/components/MediaDevicesPreview.vue +++ b/src/components/MediaDevicesPreview.vue @@ -331,7 +331,7 @@ export default { this.audioStreamError = null - if (!this.audioInputId) { + if (this.audioInputId === null || this.audioInputId === undefined) { return } @@ -377,7 +377,7 @@ export default { this.videoStreamError = null - if (!this.videoInputId) { + if (this.videoInputId === null || this.videoInputId === undefined) { return } diff --git a/src/utils/webrtc/MediaDevicesManager.js b/src/utils/webrtc/MediaDevicesManager.js index 53cd4b236a5..a8a8e0bf35c 100644 --- a/src/utils/webrtc/MediaDevicesManager.js +++ b/src/utils/webrtc/MediaDevicesManager.js @@ -41,9 +41,13 @@ * limitations (for example, currently Firefox does not list "audiooutput" * devices). * - * The label may not be available if persistent media permissions have not been - * granted and a MediaStream has not been active. In those cases the fallback - * label can be used instead. + * In some browsers if persistent media permissions have not been granted and a + * MediaStream is not active the list may contain at most one device of each + * kind, and all of them with empty attributes except for the kind. + * + * In other browsers just the label may not be available if persistent media + * permissions have not been granted and a MediaStream has not been active. In + * those cases the fallback label can be used instead. * * "attributes.audioInputId" and "attributes.videoInputId" define the devices * that will be used when calling "getUserMedia(constraints)". @@ -191,14 +195,14 @@ MediaDevicesManager.prototype = { } else { // Generate a fallback label to be used when the actual label is // not available. - if (addedDevice.deviceId === 'default') { + if (addedDevice.deviceId === 'default' || addedDevice.deviceId === '') { addedDevice.fallbackLabel = t('spreed', 'Default') } else if (addedDevice.kind === 'audioinput') { - addedDevice.fallbackLabel = t('spreed', 'Microphone {number}', { number: Object.values(this._knownDevices).filter(device => device.kind === 'audioinput').length + 1 }) + addedDevice.fallbackLabel = t('spreed', 'Microphone {number}', { number: Object.values(this._knownDevices).filter(device => device.kind === 'audioinput' && device.deviceId !== '').length + 1 }) } else if (addedDevice.kind === 'videoinput') { - addedDevice.fallbackLabel = t('spreed', 'Camera {number}', { number: Object.values(this._knownDevices).filter(device => device.kind === 'videoinput').length + 1 }) + addedDevice.fallbackLabel = t('spreed', 'Camera {number}', { number: Object.values(this._knownDevices).filter(device => device.kind === 'videoinput' && device.deviceId !== '').length + 1 }) } else if (addedDevice.kind === 'audiooutput') { - addedDevice.fallbackLabel = t('spreed', 'Speaker {number}', { number: Object.values(this._knownDevices).filter(device => device.kind === 'audioutput').length + 1 }) + addedDevice.fallbackLabel = t('spreed', 'Speaker {number}', { number: Object.values(this._knownDevices).filter(device => device.kind === 'audioutput' && device.deviceId !== '').length + 1 }) } }