feat(react): Support for Background Filters and Background Blurring (#1283)

Adds API for registering video filters, default plugin based on TensorFlow Lite, and a sample integration in our Demo app.

**New APIs:**
- `camera.registerFilter(async (mediaStream) => mediaStream)` the core
part. It allows you to chain multiple video and audio filters.
- `@stream-io/video-filters-web` - a new lib that is responsible for the
ML part and WebGL rendering
- `BackgroundFiltersProvider` - a component in the React SDK that helps
you to manage the applied filters.

Our dogfooding app integrates these new APIs and provides the necessary input (background images) and styling.

### Notes
Because of our deployment setup, this PR contains the changes done as part of #1271 and #1276 :)

Author: oliverlaz

.github/workflows/deploy-react-sample-apps.yml

@@ -47,6 +47,8 @@ jobs:
             project-id: prj_0WnHcvVkXpM4PRc2ymVmrAHFILoT
           - name: react-dogfood - https://pronto.getstream.io
             project-id: prj_4TTdjeVHEDhWWiFRfjIr1QFb5ell
+          - name: react-dogfood-staging - https://pronto-staging.getstream.io
+            project-id: prj_xmKoy4ExySCQ4fpWoWSVXlcuvtfS
           - name: react-dogfood - https://getstream.io/video/demos
             project-id: prj_tTLn3XMVal4D1Nnel9nPJ0hoI9wA
             base-path: /video/demos

package.json

@@ -25,7 +25,8 @@
     "start:styling": "yarn workspace @stream-io/video-styling run start",
     "build:client": "yarn workspace @stream-io/video-client run build",
     "start:client": "yarn workspace @stream-io/video-client run start",
-    "build:react:deps": "yarn build:client && yarn build:styling && yarn build:react:bindings && yarn build:react:sdk",
+    "build:video-filters-web": "yarn workspace @stream-io/video-filters-web run build",
+    "build:react:deps": "yarn build:client && yarn build:styling && yarn build:react:bindings && yarn build:video-filters-web && yarn build:react:sdk",
     "build:react-native:deps": "yarn build:client && yarn build:react:bindings && yarn build:react-native:sdk",
     "build:vercel": "yarn build:react:deps && yarn build:react:dogfood",
     "start:egress": "yarn workspace @stream-io/egress-composite start",
@@ -51,7 +52,9 @@
     "release": "nx run-many --target version --parallel=1",
     "release:react-bindings": "yarn workspace @stream-io/video-react-bindings npm publish --access=public",
     "release:react-sdk": "yarn workspace @stream-io/video-react-sdk npm publish --access=public",
-    "release:react-native-sdk": "yarn workspace @stream-io/video-react-native-sdk npm publish --access=public"
+    "release:react-native-sdk": "yarn workspace @stream-io/video-react-native-sdk npm publish --access=public",
+    "release:video-filters-web": "yarn workspace @stream-io/video-filters-web npm publish --access=public",
+    "release:styling": "yarn workspace @stream-io/video-styling npm publish --access=public"
   },
   "devDependencies": {
     "@commitlint/cli": "^17.0.0",
@@ -70,8 +73,8 @@
     "ngx-deploy-npm": "^5.2.0",
     "nx": "16.0.1",
     "prettier": "^2.8.8",
-    "typescript": "^5.2.2",
-    "vercel": "^33.2.0",
+    "typescript": "^5.4.3",
+    "vercel": "^33.6.2",
     "vite": "^4.4.11"
   }
 }

packages/client/docusaurus/docs/javascript/02-guides/04-camera-and-microphone.mdx

@@ -248,6 +248,49 @@ call.microphone.state.speakingWhileMuted$.subscribe((isSpeaking) => {
 
 The notification is automatically disabled if the user doesn't have the permission to send audio.
 
+## Camera and Microphone Filters
+
+Both the Camera and the Microphone allow you to apply filters to the media stream.
+This can be useful for various use-cases, such as:
+
+- applying a video effects such as background blurring, or background replacement
+- applying a custom video filter (e.g. color correction, or face detection)
+- applying a custom audio filter (e.g. noise reduction)
+
+```typescript
+import { Call } from '@stream-io/video-client';
+
+let call: Call;
+
+// apply a custom video filter
+const unregisterMyVideoFilter = await camera.registerFilter(
+  async function myVideoFilter(inputMediaStream: MediaStream) {
+    // initialize the video filter, do some magic and
+    // return the modified media stream
+    return mediaStreamWithFilterApplied;
+  },
+);
+
+// apply a custom audio filter
+const unregisterMyAudioFilter = await microphone.registerFilter(
+  async function myAudioFilter(inputMediaStream: MediaStream) {
+    // initialize the audio filter, do some magic and
+    // return the modified media stream
+    return mediaStreamWithFilterApplied;
+  },
+);
+
+// unregister the filters
+unregisterMyVideoFilter();
+unregisterMyAudioFilter();
+```
+
+Filters can be registered and unregistered at any time, and the SDK will take care of the rest.
+Filters can be chained, and the order of registration matters.
+The first registered filter will be the first to modify the raw `MediaStream`.
+
+Once a filter(s) is registered, the SDK will send the last returned `MediaStream` to the remote participants.
+
 ## Speaker management
 
 ### List and select devices

packages/client/generate-openapi.sh

@@ -10,8 +10,10 @@ else
 fi
 if  [ "$FROM_REPO" == 'chat' ]; then
   SCHEMA_FILE="$PROTOCOL_REPO_DIR/releases/video-openapi.yaml"
-else
+elif [ "$FROM_REPO" == 'protocol' ]; then
   SCHEMA_FILE="$PROTOCOL_REPO_DIR/openapi/video-openapi.yaml"
+else
+  SCHEMA_FILE=$FROM_REPO
 fi
 
 if  [ "$FROM_REPO" == 'chat' ]; then

packages/client/package.json

@@ -44,8 +44,8 @@
   },
   "devDependencies": {
     "@openapitools/openapi-generator-cli": "^2.7.0",
-    "@rollup/plugin-replace": "^5.0.4",
-    "@rollup/plugin-typescript": "^11.1.5",
+    "@rollup/plugin-replace": "^5.0.5",
+    "@rollup/plugin-typescript": "^11.1.6",
     "@types/jsonwebtoken": "^9.0.3",
     "@types/sdp-transform": "^2.4.7",
     "@types/ua-parser-js": "^0.7.37",
@@ -55,7 +55,7 @@
     "prettier": "^2.8.8",
     "rimraf": "^5.0.5",
     "rollup": "^3.29.4",
-    "typescript": "^5.2.2",
+    "typescript": "^5.4.3",
     "vite": "^4.4.11",
     "vitest": "^0.34.4",
     "vitest-mock-extended": "^1.2.1"

packages/client/src/Call.ts

@@ -30,6 +30,7 @@ import {
   GoLiveRequest,
   GoLiveResponse,
   ListRecordingsResponse,
+  ListTranscriptionsResponse,
   MuteUsersRequest,
   MuteUsersResponse,
   OwnCapability,
@@ -48,10 +49,13 @@ import {
   StartHLSBroadcastingResponse,
   StartRecordingRequest,
   StartRecordingResponse,
+  StartTranscriptionRequest,
+  StartTranscriptionResponse,
   StatsOptions,
   StopHLSBroadcastingResponse,
   StopLiveResponse,
   StopRecordingResponse,
+  StopTranscriptionResponse,
   UnblockUserRequest,
   UnblockUserResponse,
   UnpinRequest,
@@ -509,10 +513,10 @@ export class Call {
 
     this.clientStore.unregisterCall(this);
 
-    this.camera.removeSubscriptions();
-    this.microphone.removeSubscriptions();
-    this.screenShare.removeSubscriptions();
-    this.speaker.removeSubscriptions();
+    this.camera.dispose();
+    this.microphone.dispose();
+    this.screenShare.dispose();
+    this.speaker.dispose();
 
     const stopOnLeavePromises: Promise<void>[] = [];
     if (this.camera.stopOnLeave) {
@@ -1582,6 +1586,29 @@ export class Call {
     );
   };
 
+  /**
+   * Starts the transcription of the call.
+   *
+   * @param request the request data.
+   */
+  startTranscription = async (
+    request?: StartTranscriptionRequest,
+  ): Promise<StartTranscriptionResponse> => {
+    return this.streamClient.post<
+      StartTranscriptionResponse,
+      StartTranscriptionRequest
+    >(`${this.streamClientBasePath}/start_transcription`, request);
+  };
+
+  /**
+   * Stops the transcription of the call.
+   */
+  stopTranscription = async (): Promise<StopTranscriptionResponse> => {
+    return this.streamClient.post<StopTranscriptionResponse>(
+      `${this.streamClientBasePath}/stop_transcription`,
+    );
+  };
+
   /**
    * Sends a `call.permission_request` event to all users connected to the call. The call settings object contains infomration about which permissions can be requested during a call (for example a user might be allowed to request permission to publish audio, but not video).
    */
@@ -1859,6 +1886,17 @@ export class Call {
     );
   };
 
+  /**
+   * Retrieves the list of transcriptions for the current call.
+   *
+   * @returns the list of transcriptions.
+   */
+  queryTranscriptions = async (): Promise<ListTranscriptionsResponse> => {
+    return this.streamClient.get<ListTranscriptionsResponse>(
+      `${this.streamClientBasePath}/transcriptions`,
+    );
+  };
+
   /**
    * Retrieve call statistics for a particular call session (historical).
    * Here `callSessionID` is mandatory.