Fix Chromium performance hit in calls due to blur filter

lib/Listener/CSPListener.php

@@ -50,6 +50,8 @@ public function handle(Event $event): void {
 			$csp->addAllowedConnectDomain($server);
 		}
 
+		$csp->addAllowedWorkerSrcDomain('\'self\'');
+
 		$event->addPolicy($csp);
 	}
 }

src/components/CallView/Grid/Grid.vue

@@ -426,7 +426,7 @@ export default {
 
 		// Blur radius for each background in the grid
 		videoBackgroundBlur() {
-			return this.$store.getters.getBlurFilter(this.videoWidth, this.videoHeight)
+			return this.$store.getters.getBlurRadius(this.videoWidth, this.videoHeight)
 		},
 
 		stripeOpen() {

src/components/CallView/shared/Video.vue

@@ -159,8 +159,8 @@ export default {
 		},
 		// Calculated once in the grid component for each video background
 		videoBackgroundBlur: {
-			type: String,
-			default: '',
+			type: Number,
+			default: 0,
 		},
 	},
 

src/components/CallView/shared/VideoBackground.vue

@@ -25,14 +25,15 @@
 			ref="darkener"
 			class="darken">
 			<ResizeObserver
-				v-if="gridBlur === ''"
+				v-if="gridBlur === 0"
 				class="observer"
 				@notify="setBlur" />
 		</div>
 		<img
 			v-if="hasPicture"
+			ref="backgroundImage"
 			:src="backgroundImage"
-			:style="gridBlur ? gridBlur : blur"
+			:style="backgroundStyle"
 			class="video-background__picture"
 			alt="">
 		<div v-else
@@ -47,6 +48,8 @@ import usernameToColor from '@nextcloud/vue/dist/Functions/usernameToColor'
 import { generateUrl } from '@nextcloud/router'
 import { ResizeObserver } from 'vue-resize'
 import { getBuilder } from '@nextcloud/browser-storage'
+import browserCheck from '../../../mixins/browserCheck'
+import blur from '../../../utils/imageBlurrer'
 
 const browserStorage = getBuilder('nextcloud').persist().build()
 
@@ -69,6 +72,10 @@ export default {
 		ResizeObserver,
 	},
 
+	mixins: [
+		browserCheck,
+	],
+
 	props: {
 		displayName: {
 			type: String,
@@ -79,15 +86,21 @@ export default {
 			default: '',
 		},
 		gridBlur: {
-			type: String,
-			default: '',
+			type: Number,
+			default: 0,
 		},
 	},
 
 	data() {
 		return {
 			hasPicture: false,
-			blur: '',
+			useCssBlurFilter: true,
+			blur: 0,
+			blurredBackgroundImage: null,
+			blurredBackgroundImageCache: {},
+			blurredBackgroundImageSource: null,
+			pendingGenerateBlurredBackgroundImageCount: 0,
+			isDestroyed: false,
 		}
 	},
 
@@ -103,11 +116,78 @@ export default {
 			}
 		},
 		backgroundImage() {
+			return this.useCssBlurFilter ? this.backgroundImageUrl : this.blurredBackgroundImage
+		},
+		backgroundImageUrl() {
+			if (!this.user) {
+				return null
+			}
+
 			return generateUrl(`avatar/${this.user}/300`)
 		},
+		backgroundBlur() {
+			return this.gridBlur ? this.gridBlur : this.blur
+		},
+		backgroundStyle() {
+			if (!this.useCssBlurFilter) {
+				return {}
+			}
+
+			return {
+				filter: `blur(${this.backgroundBlur}px)`,
+			}
+		},
+		// Special computed property to combine the properties that should be
+		// watched to generate (or not) the blurred background image.
+		generatedBackgroundBlur() {
+			if (!this.hasPicture || this.useCssBlurFilter) {
+				return false
+			}
+
+			if (!this.blurredBackgroundImageSource) {
+				return false
+			}
+
+			return this.backgroundBlur
+		},
+	},
+
+	watch: {
+		backgroundImageUrl: {
+			immediate: true,
+			handler() {
+				this.blurredBackgroundImageSource = null
+
+				if (!this.backgroundImageUrl) {
+					return
+				}
+
+				const image = new Image()
+				image.onload = () => {
+					createImageBitmap(image).then(imageBitmap => {
+						this.blurredBackgroundImageSource = imageBitmap
+					})
+				}
+				image.src = this.backgroundImageUrl
+			},
+		},
+		generatedBackgroundBlur: {
+			immediate: true,
+			handler() {
+				if (this.generatedBackgroundBlur === false) {
+					return
+				}
+
+				this.generateBlurredBackgroundImage()
+			},
+		},
 	},
 
 	async beforeMount() {
+		if (this.isChrome) {
+			this.useCssBlurFilter = false
+		}
+
 		if (!this.user) {
 			return
 		}
@@ -142,10 +222,68 @@ export default {
 		}
 	},
 
+	beforeDestroy() {
+		this.isDestroyed = true
+	},
+
 	methods: {
 		// Calculate the background blur based on the height of the background element
 		setBlur({ width, height }) {
-			this.blur = this.$store.getters.getBlurFilter(width, height)
+			this.blur = this.$store.getters.getBlurRadius(width, height)
+		},
+
+		generateBlurredBackgroundImage() {
+			// Reset image source so the width and height are adjusted to
+			// the element rather than to the previous image being shown.
+			this.$refs.backgroundImage.src = ''
+
+			let width = this.$refs.backgroundImage.width
+			let height = this.$refs.backgroundImage.height
+
+			// Restore the current background so it is shown instead of an empty
+			// background while the new one is being generated.
+			this.$refs.backgroundImage.src = this.blurredBackgroundImage
+
+			const sourceAspectRatio = this.blurredBackgroundImageSource.width / this.blurredBackgroundImageSource.height
+			const canvasAspectRatio = width / height
+
+			if (canvasAspectRatio > sourceAspectRatio) {
+				height = width / sourceAspectRatio
+			} else if (canvasAspectRatio < sourceAspectRatio) {
+				width = height * sourceAspectRatio
+			}
+
+			const cacheId = this.backgroundImageUrl + '-' + width + '-' + height + '-' + this.backgroundBlur
+			if (this.blurredBackgroundImageCache[cacheId]) {
+				this.blurredBackgroundImage = this.blurredBackgroundImageCache[cacheId]
+
+				return
+			}
+
+			if (this.pendingGenerateBlurredBackgroundImageCount) {
+				this.pendingGenerateBlurredBackgroundImageCount++
+
+				return
+			}
+
+			this.pendingGenerateBlurredBackgroundImageCount = 1
+
+			blur(this.blurredBackgroundImageSource, width, height, this.backgroundBlur).then(image => {
+				if (this.isDestroyed) {
+					return
+				}
+
+				this.blurredBackgroundImage = image
+				this.blurredBackgroundImageCache[cacheId] = this.blurredBackgroundImage
+
+				const generateBlurredBackgroundImageCalledAgain = this.pendingGenerateBlurredBackgroundImageCount > 1
+
+				this.pendingGenerateBlurredBackgroundImageCount = 0
+
+				if (generateBlurredBackgroundImageCalledAgain) {
+					this.generateBlurredBackgroundImage()
+				}
+			})
 		},
 	},
 }

src/store/callViewStore.js

@@ -44,8 +44,12 @@ const getters = {
 	selectedVideoPeerId: (state) => {
 		return state.selectedVideoPeerId
 	},
-	getBlurFilter: (state) => (width, height) => {
-		return `filter: blur(${(width * height * state.videoBackgroundBlur) / 1000}px)`
+	/**
+	 * @param {object} state the width and height to calculate the radius from
+	 * @returns {number} the blur radius to use, in pixels
+	 */
+	getBlurRadius: (state) => (width, height) => {
+		return (width * height * state.videoBackgroundBlur) / 1000
 	},
 }
 

src/utils/imageBlurrer.js

@@ -0,0 +1,76 @@
+/**
+ *
+ * @copyright Copyright (c) 2020, Daniel Calviño Sánchez ([email protected])
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+import { generateFilePath } from '@nextcloud/router'
+
+const worker = new Worker(generateFilePath('spreed', '', 'js/image-blurrer-worker.js'))
+
+const pendingResults = {}
+let pendingResultsNextId = 0
+
+worker.onmessage = function(message) {
+	const pendingResult = pendingResults[message.data.id]
+	if (!pendingResult) {
+		console.debug('No pending result for blurring image with id ' + message.data.id)
+
+		return
+	}
+
+	pendingResult(message.data.blurredImageAsDataUrl)
+
+	delete pendingResults[message.data.id]
+}
+
+function blurSync(image, width, height, blurRadius) {
+	return new Promise((resolve, reject) => {
+		const canvas = document.createElement('canvas')
+		canvas.width = width
+		canvas.height = height
+
+		const context = canvas.getContext('2d')
+		context.filter = `blur(${blurRadius}px)`
+		context.drawImage(image, 0, 0, canvas.width, canvas.height)
+
+		resolve(canvas.toDataURL())
+	})
+}
+
+export default function blur(image, width, height, blurRadius) {
+	if (typeof OffscreenCanvas === 'undefined') {
+		return blurSync(image, width, height, blurRadius)
+	}
+
+	const id = pendingResultsNextId
+
+	pendingResultsNextId++
+
+	return new Promise((resolve, reject) => {
+		pendingResults[id] = resolve
+
+		worker.postMessage({
+			id: id,
+			image: image,
+			width: width,
+			height: height,
+			blurRadius: blurRadius,
+		})
+	})
+}

src/utils/imageBlurrerWorker.js

@@ -0,0 +1,37 @@
+/**
+ *
+ * @copyright Copyright (c) 2020, Daniel Calviño Sánchez ([email protected])
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+const fileReaderSync = new global.FileReaderSync()
+
+onmessage = function(message) {
+	const offscreenCanvas = new OffscreenCanvas(message.data.width, message.data.height)
+
+	const context = offscreenCanvas.getContext('2d')
+	context.filter = `blur(${message.data.blurRadius}px)`
+	context.drawImage(message.data.image, 0, 0, offscreenCanvas.width, offscreenCanvas.height)
+
+	offscreenCanvas.convertToBlob().then(blob => {
+		postMessage({
+			id: message.data.id,
+			blurredImageAsDataUrl: fileReaderSync.readAsDataURL(blob),
+		})
+	})
+}

webpack.common.js

@@ -7,6 +7,11 @@ module.exports = {
 	entry: {
 		'admin-settings': path.join(__dirname, 'src', 'mainAdminSettings.js'),
 		'collections': path.join(__dirname, 'src', 'collections.js'),
+		// There is a "worker-loader" plugin for Webpack, but I was not able to
+		// get it to work ("publicPath" uses "output.publicPath" rather than the
+		// one set in the plugin
+		// https://github.com/webpack-contrib/worker-loader/issues/281).
+		'image-blurrer-worker': path.join(__dirname, 'src', 'utils/imageBlurrerWorker.js'),
 		'talk': path.join(__dirname, 'src', 'main.js'),
 		'talk-files-sidebar': [
 			path.join(__dirname, 'src', 'mainFilesSidebar.js'),