Add support for simulcast streams.

src/components/CallView/CallView.vue

@@ -159,6 +159,7 @@
 
 <script>
 import Grid from './Grid/Grid'
+import { SIMULCAST } from '../../constants'
 import { localMediaModel, localCallParticipantModel, callParticipantCollection } from '../../utils/webrtc/index'
 import { fetchPeers } from '../../services/callsService'
 import { showMessage } from '@nextcloud/dialogs'
@@ -344,6 +345,14 @@ export default {
 			this.updateDataFromCallParticipantModels(models)
 		},
 
+		isGrid() {
+			this.adjustSimulcastQuality()
+		},
+
+		selectedVideoPeerId() {
+			this.adjustSimulcastQuality()
+		},
+
 		speakers() {
 			this._setPromotedParticipant()
 		},
@@ -472,6 +481,8 @@ export default {
 				}, function(raisedHand) {
 					this._handleParticipantRaisedHand(addedModel, raisedHand)
 				})
+
+				this.adjustSimulcastQualityForParticipant(addedModel)
 			})
 		},
 
@@ -553,6 +564,8 @@ export default {
 			if (!this.screenSharingActive && this.speakers.length) {
 				this.sharedDatas[this.speakers[0].id].promoted = true
 			}
+
+			this.adjustSimulcastQuality()
 		},
 
 		_switchScreenToId(id) {
@@ -633,6 +646,22 @@ export default {
 		handleToggleVideo({ peerId, value }) {
 			this.sharedDatas[peerId].videoEnabled = value
 		},
+
+		adjustSimulcastQuality() {
+			this.callParticipantModels.forEach(callParticipantModel => {
+				this.adjustSimulcastQualityForParticipant(callParticipantModel)
+			})
+		},
+
+		adjustSimulcastQualityForParticipant(callParticipantModel) {
+			if (this.isGrid) {
+				callParticipantModel.setSimulcastVideoQuality(SIMULCAST.MEDIUM)
+			} else if (this.sharedDatas[callParticipantModel.attributes.peerId].promoted || this.selectedVideoPeerId === callParticipantModel.attributes.peerId) {
+				callParticipantModel.setSimulcastVideoQuality(SIMULCAST.HIGH)
+			} else {
+				callParticipantModel.setSimulcastVideoQuality(SIMULCAST.LOW)
+			}
+		},
 	},
 }
 </script>

src/constants.js

@@ -124,3 +124,8 @@ export const PRIVACY = {
 	PUBLIC: 0,
 	PRIVATE: 1,
 }
+export const SIMULCAST = {
+	LOW: 0,
+	MEDIUM: 1,
+	HIGH: 2,
+}

src/utils/webrtc/models/CallParticipantModel.js

@@ -355,4 +355,22 @@ CallParticipantModel.prototype = {
 		this.set('nextcloudSessionId', nextcloudSessionId)
 	},
 
+	setSimulcastVideoQuality(simulcastVideoQuality) {
+		if (!this.get('peer') || !this.get('peer').enableSimulcast) {
+			return
+		}
+
+		// Use same quality for simulcast and temporal layer.
+		this.get('peer').selectSimulcastStream(simulcastVideoQuality, simulcastVideoQuality)
+	},
+
+	setSimulcastScreenQuality(simulcastScreenQuality) {
+		if (!this.get('screenPeer') || !this.get('screenPeer').enableSimulcast) {
+			return
+		}
+
+		// Use same quality for simulcast and temporal layer.
+		this.get('screenPeer').selectSimulcastStream(simulcastScreenQuality, simulcastScreenQuality)
+	},
+
 }

src/utils/webrtc/simplewebrtc/peer.js

@@ -2,6 +2,7 @@
 const initialState = require('@nextcloud/initial-state')
 const sdpTransform = require('sdp-transform')
 
+const adapter = require('webrtc-adapter')
 const util = require('util')
 const webrtcSupport = require('webrtcsupport')
 const WildEmitter = require('wildemitter')
@@ -28,6 +29,8 @@ function Peer(options) {
 	this.stream = options.stream
 	this.sendVideoIfAvailable = options.sendVideoIfAvailable === undefined ? true : options.sendVideoIfAvailable
 	this.enableDataChannels = options.enableDataChannels === undefined ? this.parent.config.enableDataChannels : options.enableDataChannels
+	this.enableSimulcast = options.enableSimulcast === undefined ? this.parent.config.enableSimulcast : options.enableSimulcast
+	this.maxBitrates = options.maxBitrates === undefined ? this.parent.config.maxBitrates : options.maxBitrates
 	this.receiveMedia = options.receiveMedia || this.parent.config.receiveMedia
 	this.channels = {}
 	this.pendingDCMessages = [] // key (datachannel label) -> value (array[pending messages])
@@ -184,12 +187,260 @@ function preferH264VideoCodecIfAvailable(sessionDescription) {
 	return sessionDescription
 }
 
+// Helper method to munge an SDP to enable simulcasting (Chrome only)
+// Taken from janus.js (MIT license).
+/* eslint-disable */
+function mungeSdpForSimulcasting(sdp) {
+	// Let's munge the SDP to add the attributes for enabling simulcasting
+	// (based on https://gist.github.com/ggarber/a19b4c33510028b9c657)
+	var lines = sdp.split("\r\n");
+	var video = false;
+	var ssrc = [ -1 ], ssrc_fid = [ -1 ];
+	var cname = null, msid = null, mslabel = null, label = null;
+	var insertAt = -1;
+	for(var i=0; i<lines.length; i++) {
+		var mline = lines[i].match(/m=(\w+) */);
+		if(mline) {
+			var medium = mline[1];
+			if(medium === "video") {
+				// New video m-line: make sure it's the first one
+				if(ssrc[0] < 0) {
+					video = true;
+				} else {
+					// We're done, let's add the new attributes here
+					insertAt = i;
+					break;
+				}
+			} else {
+				// New non-video m-line: do we have what we were looking for?
+				if(ssrc[0] > -1) {
+					// We're done, let's add the new attributes here
+					insertAt = i;
+					break;
+				}
+			}
+			continue;
+		}
+		if(!video)
+			continue;
+		var fid = lines[i].match(/a=ssrc-group:FID (\d+) (\d+)/);
+		if(fid) {
+			ssrc[0] = fid[1];
+			ssrc_fid[0] = fid[2];
+			lines.splice(i, 1); i--;
+			continue;
+		}
+		if(ssrc[0]) {
+			var match = lines[i].match('a=ssrc:' + ssrc[0] + ' cname:(.+)')
+			if(match) {
+				cname = match[1];
+			}
+			match = lines[i].match('a=ssrc:' + ssrc[0] + ' msid:(.+)')
+			if(match) {
+				msid = match[1];
+			}
+			match = lines[i].match('a=ssrc:' + ssrc[0] + ' mslabel:(.+)')
+			if(match) {
+				mslabel = match[1];
+			}
+			match = lines[i].match('a=ssrc:' + ssrc[0] + ' label:(.+)')
+			if(match) {
+				label = match[1];
+			}
+			if(lines[i].indexOf('a=ssrc:' + ssrc_fid[0]) === 0) {
+				lines.splice(i, 1); i--;
+				continue;
+			}
+			if(lines[i].indexOf('a=ssrc:' + ssrc[0]) === 0) {
+				lines.splice(i, 1); i--;
+				continue;
+			}
+		}
+		if(lines[i].length == 0) {
+			lines.splice(i, 1); i--;
+			continue;
+		}
+	}
+	if(ssrc[0] < 0) {
+		// Couldn't find a FID attribute, let's just take the first video SSRC we find
+		insertAt = -1;
+		video = false;
+		for(var i=0; i<lines.length; i++) {
+			var mline = lines[i].match(/m=(\w+) */);
+			if(mline) {
+				var medium = mline[1];
+				if(medium === "video") {
+					// New video m-line: make sure it's the first one
+					if(ssrc[0] < 0) {
+						video = true;
+					} else {
+						// We're done, let's add the new attributes here
+						insertAt = i;
+						break;
+					}
+				} else {
+					// New non-video m-line: do we have what we were looking for?
+					if(ssrc[0] > -1) {
+						// We're done, let's add the new attributes here
+						insertAt = i;
+						break;
+					}
+				}
+				continue;
+			}
+			if(!video)
+				continue;
+			if(ssrc[0] < 0) {
+				var value = lines[i].match(/a=ssrc:(\d+)/);
+				if(value) {
+					ssrc[0] = value[1];
+					lines.splice(i, 1); i--;
+					continue;
+				}
+			} else {
+				var match = lines[i].match('a=ssrc:' + ssrc[0] + ' cname:(.+)')
+				if(match) {
+					cname = match[1];
+				}
+				match = lines[i].match('a=ssrc:' + ssrc[0] + ' msid:(.+)')
+				if(match) {
+					msid = match[1];
+				}
+				match = lines[i].match('a=ssrc:' + ssrc[0] + ' mslabel:(.+)')
+				if(match) {
+					mslabel = match[1];
+				}
+				match = lines[i].match('a=ssrc:' + ssrc[0] + ' label:(.+)')
+				if(match) {
+					label = match[1];
+				}
+				if(lines[i].indexOf('a=ssrc:' + ssrc_fid[0]) === 0) {
+					lines.splice(i, 1); i--;
+					continue;
+				}
+				if(lines[i].indexOf('a=ssrc:' + ssrc[0]) === 0) {
+					lines.splice(i, 1); i--;
+					continue;
+				}
+			}
+			if(lines[i].length === 0) {
+				lines.splice(i, 1); i--;
+				continue;
+			}
+		}
+	}
+	if(ssrc[0] < 0) {
+		// Still nothing, let's just return the SDP we were asked to munge
+		console.warn("Couldn't find the video SSRC, simulcasting NOT enabled");
+		return sdp;
+	}
+	if(insertAt < 0) {
+		// Append at the end
+		insertAt = lines.length;
+	}
+	// Generate a couple of SSRCs (for retransmissions too)
+	// Note: should we check if there are conflicts, here?
+	ssrc[1] = Math.floor(Math.random()*0xFFFFFFFF);
+	ssrc[2] = Math.floor(Math.random()*0xFFFFFFFF);
+	ssrc_fid[1] = Math.floor(Math.random()*0xFFFFFFFF);
+	ssrc_fid[2] = Math.floor(Math.random()*0xFFFFFFFF);
+	// Add attributes to the SDP
+	for(var i=0; i<ssrc.length; i++) {
+		if(cname) {
+			lines.splice(insertAt, 0, 'a=ssrc:' + ssrc[i] + ' cname:' + cname);
+			insertAt++;
+		}
+		if(msid) {
+			lines.splice(insertAt, 0, 'a=ssrc:' + ssrc[i] + ' msid:' + msid);
+			insertAt++;
+		}
+		if(mslabel) {
+			lines.splice(insertAt, 0, 'a=ssrc:' + ssrc[i] + ' mslabel:' + mslabel);
+			insertAt++;
+		}
+		if(label) {
+			lines.splice(insertAt, 0, 'a=ssrc:' + ssrc[i] + ' label:' + label);
+			insertAt++;
+		}
+		// Add the same info for the retransmission SSRC
+		if(cname) {
+			lines.splice(insertAt, 0, 'a=ssrc:' + ssrc_fid[i] + ' cname:' + cname);
+			insertAt++;
+		}
+		if(msid) {
+			lines.splice(insertAt, 0, 'a=ssrc:' + ssrc_fid[i] + ' msid:' + msid);
+			insertAt++;
+		}
+		if(mslabel) {
+			lines.splice(insertAt, 0, 'a=ssrc:' + ssrc_fid[i] + ' mslabel:' + mslabel);
+			insertAt++;
+		}
+		if(label) {
+			lines.splice(insertAt, 0, 'a=ssrc:' + ssrc_fid[i] + ' label:' + label);
+			insertAt++;
+		}
+	}
+	lines.splice(insertAt, 0, 'a=ssrc-group:FID ' + ssrc[2] + ' ' + ssrc_fid[2]);
+	lines.splice(insertAt, 0, 'a=ssrc-group:FID ' + ssrc[1] + ' ' + ssrc_fid[1]);
+	lines.splice(insertAt, 0, 'a=ssrc-group:FID ' + ssrc[0] + ' ' + ssrc_fid[0]);
+	lines.splice(insertAt, 0, 'a=ssrc-group:SIM ' + ssrc[0] + ' ' + ssrc[1] + ' ' + ssrc[2]);
+	sdp = lines.join("\r\n");
+	if(!sdp.endsWith("\r\n"))
+		sdp += "\r\n";
+	return sdp;
+}
+/* eslint-enable */
+
 Peer.prototype.offer = function(options) {
+	const sendVideo = this.sendVideoIfAvailable && this.type !== 'screen'
+	if (sendVideo && this.enableSimulcast && adapter.browserDetails.browser === 'firefox') {
+		console.debug('Enabling Simulcasting for Firefox (RID)')
+		const sender = this.pc.getSenders().find(function(s) {
+			return s.track.kind === 'video'
+		})
+		if (sender) {
+			let parameters = sender.getParameters()
+			if (!parameters) {
+				parameters = {}
+			}
+			parameters.encodings = [
+				{
+					rid: 'h',
+					active: true,
+					maxBitrate: this.maxBitrates.high,
+				},
+				{
+					rid: 'm',
+					active: true,
+					maxBitrate: this.maxBitrates.medium,
+					scaleResolutionDownBy: 2,
+				},
+				{
+					rid: 'l',
+					active: true,
+					maxBitrate: this.maxBitrates.low,
+					scaleResolutionDownBy: 4,
+				},
+			]
+			sender.setParameters(parameters)
+		}
+	}
 	this.pc.createOffer(options).then(function(offer) {
 		if (shouldPreferH264()) {
 			console.debug('Preferring hardware codec H.264 as per global configuration')
 			offer = preferH264VideoCodecIfAvailable(offer)
 		}
+
+		if (sendVideo && this.enableSimulcast) {
+			// This SDP munging only works with Chrome (Safari STP may support it too)
+			if (adapter.browserDetails.browser === 'chrome' || adapter.browserDetails.browser === 'safari') {
+				console.debug('Enabling Simulcasting for Chrome (SDP munging)')
+				offer.sdp = mungeSdpForSimulcasting(offer.sdp)
+			} else if (adapter.browserDetails.browser !== 'firefox') {
+				console.debug('Simulcast can only be enabled on Chrome or Firefox')
+			}
+		}
+
 		this.pc.setLocalDescription(offer).then(function() {
 			if (this.parent.config.nick) {
 				// The offer is a RTCSessionDescription that only serializes
@@ -250,6 +501,21 @@ Peer.prototype.handleAnswer = function(answer) {
 	})
 }
 
+Peer.prototype.selectSimulcastStream = function(substream, temporal) {
+	if (this.substream === substream && this.temporal === temporal) {
+		console.debug('Simulcast stream not changed', this, substream, temporal)
+		return
+	}
+
+	console.debug('Changing simulcast stream', this, substream, temporal)
+	this.send('selectStream', {
+		substream,
+		temporal,
+	})
+	this.substream = substream
+	this.temporal = temporal
+}
+
 Peer.prototype.handleMessage = function(message) {
 	const self = this