From 8da7dc2371e46531262d6eaa17d0db9409dae913 Mon Sep 17 00:00:00 2001 From: chriszarate Date: Thu, 24 Jul 2025 14:28:59 -0400 Subject: [PATCH 01/54] Add `core.getSyncProvider` filter This filter allows external code to load its own sync provider. This can be used by both Gutenberg itself and plugins to customize the sync transport. --- lib/client-assets.php | 6 +++ package-lock.json | 2 + packages/core-data/package.json | 1 + packages/core-data/src/sync.js | 27 ------------ packages/core-data/src/sync.ts | 43 +++++++++++++++++++ packages/core-data/tsconfig.json | 1 + packages/sync/CODE.md | 22 +++++++++- packages/sync/README.md | 6 +-- packages/sync/package.json | 1 + packages/sync/src/create-webrtc-connection.js | 39 ----------------- packages/sync/src/create-webrtc-connection.ts | 40 +++++++++++++++++ packages/sync/src/index.js | 3 -- packages/sync/src/index.ts | 40 +++++++++++++++++ packages/sync/src/types.ts | 34 +++++++++++---- packages/sync/tsconfig.json | 2 +- 15 files changed, 182 insertions(+), 85 deletions(-) delete mode 100644 packages/core-data/src/sync.js create mode 100644 packages/core-data/src/sync.ts delete mode 100644 packages/sync/src/create-webrtc-connection.js create mode 100644 packages/sync/src/create-webrtc-connection.ts delete mode 100644 packages/sync/src/index.js create mode 100644 packages/sync/src/index.ts diff --git a/lib/client-assets.php b/lib/client-assets.php index e6f64106cbb323..77e5e79e96c1ab 100644 --- a/lib/client-assets.php +++ b/lib/client-assets.php @@ -212,6 +212,12 @@ function gutenberg_register_packages_scripts( $scripts ) { } break; + case 'wp-core-data': + if ( gutenberg_is_experiment_enabled( 'gutenberg-sync-collaboration' ) ) { + array_push( $dependencies, 'wp-sync' ); + } + break; + case 'wp-edit-post': array_push( $dependencies, 'media-models', 'media-views', 'postbox' ); break; diff --git a/package-lock.json b/package-lock.json index e1451765c43a3b..574cb822fa8784 100644 --- a/package-lock.json +++ b/package-lock.json @@ -50360,6 +50360,7 @@ "@wordpress/data": "file:../data", "@wordpress/deprecated": "file:../deprecated", "@wordpress/element": "file:../element", + "@wordpress/hooks": "file:../hooks", "@wordpress/html-entities": "file:../html-entities", "@wordpress/i18n": "file:../i18n", "@wordpress/is-shallow-equal": "file:../is-shallow-equal", @@ -52307,6 +52308,7 @@ "dependencies": { "@babel/runtime": "7.25.7", "@types/simple-peer": "^9.11.5", + "@wordpress/hooks": "file:../hooks", "@wordpress/url": "file:../url", "import-locals": "^2.0.0", "lib0": "^0.2.42", diff --git a/packages/core-data/package.json b/packages/core-data/package.json index 40139b2e84630d..cd0593c5f27052 100644 --- a/packages/core-data/package.json +++ b/packages/core-data/package.json @@ -40,6 +40,7 @@ "@wordpress/data": "file:../data", "@wordpress/deprecated": "file:../deprecated", "@wordpress/element": "file:../element", + "@wordpress/hooks": "file:../hooks", "@wordpress/html-entities": "file:../html-entities", "@wordpress/i18n": "file:../i18n", "@wordpress/is-shallow-equal": "file:../is-shallow-equal", diff --git a/packages/core-data/src/sync.js b/packages/core-data/src/sync.js deleted file mode 100644 index fdc421a6bd70e9..00000000000000 --- a/packages/core-data/src/sync.js +++ /dev/null @@ -1,27 +0,0 @@ -/** - * WordPress dependencies - */ -import { - createSyncProvider, - connectIndexDb, - createWebRTCConnection, -} from '@wordpress/sync'; - -let syncProvider; - -export function getSyncProvider() { - if ( ! syncProvider ) { - syncProvider = createSyncProvider( - connectIndexDb, - createWebRTCConnection( { - signaling: [ - //'ws://localhost:4444', - window?.wp?.ajax?.settings?.url, - ], - password: window?.__experimentalCollaborativeEditingSecret, - } ) - ); - } - - return syncProvider; -} diff --git a/packages/core-data/src/sync.ts b/packages/core-data/src/sync.ts new file mode 100644 index 00000000000000..f262a8508b361a --- /dev/null +++ b/packages/core-data/src/sync.ts @@ -0,0 +1,43 @@ +/** + * WordPress dependencies + */ +import { applyFilters } from '@wordpress/hooks'; + +/** + * IMPORTANT: Ensure that we only import types from `@wordpress/sync` and not + * code. The code is loaded only behind an experimental flag, which allows + * other plugins to load their own sync provider. + */ +import type { SyncProvider } from '@wordpress/sync'; + +let syncProvider: SyncProvider; + +/** + * Returns the current sync provider, filterable by external code. + * + * If no sync provider is set, it returns a fallback no-op sync provider to + * remove the need for defensive checks in the code that uses it. + * + * @return The current sync provider. + */ +export function getSyncProvider(): SyncProvider { + if ( syncProvider ) { + return syncProvider; + } + + const fallbackNoOpSyncProvider = { + __fallback: true, + bootstrap: async () => {}, + discard: async () => {}, + register: () => {}, + update: () => {}, + }; + + syncProvider = + ( applyFilters( + 'core.getSyncProvider', + null + ) as SyncProvider | null ) ?? fallbackNoOpSyncProvider; + + return syncProvider; +} diff --git a/packages/core-data/tsconfig.json b/packages/core-data/tsconfig.json index 57c9d208e4c689..31800502dae598 100644 --- a/packages/core-data/tsconfig.json +++ b/packages/core-data/tsconfig.json @@ -13,6 +13,7 @@ { "path": "../deprecated" }, { "path": "../element" }, { "path": "../html-entities" }, + { "path": "../hooks" }, { "path": "../i18n" }, { "path": "../is-shallow-equal" }, { "path": "../private-apis" }, diff --git a/packages/sync/CODE.md b/packages/sync/CODE.md index 40a4b76d2cfd42..ae27b2541716dc 100644 --- a/packages/sync/CODE.md +++ b/packages/sync/CODE.md @@ -8,9 +8,27 @@ Relevant docs: - https://github.com/WordPress/gutenberg/issues/52593 - https://docs.yjs.dev/ -## Enable the experiment +## Enable the experiments -The experiment can be enabled in the "Guteberg > Experiments" page. When it is enabled (search for `gutenberg-sync-collaboration` in the codebase), the client receives two new pieces of data: +There are two experiments that must be enabled in the "Guteberg > Experiments" page: + +- Collaboration: enable real-time collaboration +- Collaboration: WebRTC provider + +Alternatively, you can enable just the "Collaboration: enable real-time collaboration" experiment and load a custom provider (transport) via a filter: + +```js +addFilter( 'core.getSyncProvider', 'my-plugin/custom-sync-provider', ( getSyncProvider ) => { + return { + // Custom sync provider implementation + bootstrap: () => {}, + discard: () => {}, + update: () => {}, + }; +} ); +``` + +When it is enabled (search for `gutenberg-sync-collaboration` in the codebase), the client receives two new pieces of data: - `window.__experimentalEnableSync`: boolean. Used by the `core-data` package to determine whether to bootstrap and use the sync provider offered by the `sync` package. - `window.__experimentalCollaborativeEditingSecret`: string. A secret used by the `sync` package to create a secure connection among peers. diff --git a/packages/sync/README.md b/packages/sync/README.md index f15d61b5a1eb5f..6b8f984463d5b5 100644 --- a/packages/sync/README.md +++ b/packages/sync/README.md @@ -47,13 +47,11 @@ Function that creates a new WebRTC Connection. _Parameters_ -- _config_ `Object`: The object ID. -- _config.signaling_ `Array`: -- _config.password_ `string`: +- _config_ `WebRTCConnectionConfig`: Configuration for the WebRTC connection. _Returns_ -- `Function`: Promise that resolves when the connection is established. +- `ConnectDoc`: Promise that resolves when the connection is established. diff --git a/packages/sync/package.json b/packages/sync/package.json index 772979b2907c37..af982be222d900 100644 --- a/packages/sync/package.json +++ b/packages/sync/package.json @@ -31,6 +31,7 @@ "dependencies": { "@babel/runtime": "7.25.7", "@types/simple-peer": "^9.11.5", + "@wordpress/hooks": "file:../hooks", "@wordpress/url": "file:../url", "import-locals": "^2.0.0", "lib0": "^0.2.42", diff --git a/packages/sync/src/create-webrtc-connection.js b/packages/sync/src/create-webrtc-connection.js deleted file mode 100644 index 97fcddc727d024..00000000000000 --- a/packages/sync/src/create-webrtc-connection.js +++ /dev/null @@ -1,39 +0,0 @@ -/** - * External dependencies - */ -// import { WebrtcProvider } from 'y-webrtc'; - -/** - * Internal dependencies - */ -import { WebrtcProviderWithHttpSignaling } from './webrtc-http-stream-signaling'; - -/** @typedef {import('./types').ObjectType} ObjectType */ -/** @typedef {import('./types').ObjectID} ObjectID */ -/** @typedef {import('./types').CRDTDoc} CRDTDoc */ - -/** - * Function that creates a new WebRTC Connection. - * - * @param {Object} config The object ID. - * - * @param {Array} config.signaling - * @param {string} config.password - * @return {Function} Promise that resolves when the connection is established. - */ -export function createWebRTCConnection( { signaling, password } ) { - return function ( - /** @type {string} */ objectId, - /** @type {string} */ objectType, - /** @type {import("yjs").Doc} */ doc - ) { - const roomName = `${ objectType }-${ objectId }`; - new WebrtcProviderWithHttpSignaling( roomName, doc, { - signaling, - // @ts-ignore - password, - } ); - - return Promise.resolve( () => true ); - }; -} diff --git a/packages/sync/src/create-webrtc-connection.ts b/packages/sync/src/create-webrtc-connection.ts new file mode 100644 index 00000000000000..456b68694b5da2 --- /dev/null +++ b/packages/sync/src/create-webrtc-connection.ts @@ -0,0 +1,40 @@ +/** + * External dependencies + */ + +/** + * Internal dependencies + */ +import { WebrtcProviderWithHttpSignaling } from './webrtc-http-stream-signaling'; +import type { ConnectDoc, CRDTDoc, ObjectID, ObjectType } from './types'; + +export interface WebRTCConnectionConfig { + signaling: string[]; + password?: string; +} + +/** + * Function that creates a new WebRTC Connection. + * + * @param {WebRTCConnectionConfig} config Configuration for the WebRTC connection. + * @return {ConnectDoc} Promise that resolves when the connection is established. + */ +export function createWebRTCConnection( { + signaling, + password, +}: WebRTCConnectionConfig ): ConnectDoc { + return function ( + objectId: ObjectID, + objectType: ObjectType, + doc: CRDTDoc + ) { + const roomName = `${ objectType }-${ objectId }`; + new WebrtcProviderWithHttpSignaling( roomName, doc, { + signaling, + // @ts-ignore + password, + } ); + + return Promise.resolve( () => true ); + }; +} diff --git a/packages/sync/src/index.js b/packages/sync/src/index.js deleted file mode 100644 index 6c2b6899ffb618..00000000000000 --- a/packages/sync/src/index.js +++ /dev/null @@ -1,3 +0,0 @@ -export { connectIndexDb } from './connect-indexdb'; -export { createWebRTCConnection } from './create-webrtc-connection'; -export { createSyncProvider } from './provider'; diff --git a/packages/sync/src/index.ts b/packages/sync/src/index.ts new file mode 100644 index 00000000000000..3b6a625a32300e --- /dev/null +++ b/packages/sync/src/index.ts @@ -0,0 +1,40 @@ +/** + * WordPress dependencies + */ +import { addFilter } from '@wordpress/hooks'; + +/** + * Internal dependencies + */ +import { connectIndexDb } from './connect-indexdb'; +import { createWebRTCConnection } from './create-webrtc-connection'; +import { createSyncProvider } from './provider'; +import type { SyncProvider } from './types'; + +export { connectIndexDb } from './connect-indexdb'; +export { createWebRTCConnection } from './create-webrtc-connection'; +export { createSyncProvider } from './provider'; +export * from './types'; + +addFilter( + 'core.getSyncProvider', + 'wordpress-sync-webrtc/get-sync-provider', + ( provider: SyncProvider | null ): SyncProvider => { + // Do not override an already defined sync provider. + if ( provider ) { + return provider; + } + + return createSyncProvider( + connectIndexDb, + createWebRTCConnection( { + password: window?.__experimentalCollaborativeEditingSecret, + signaling: [ + //'ws://localhost:4444', + window?.wp?.ajax?.settings?.url, + ], + } ) + ); + }, + 10 +); diff --git a/packages/sync/src/types.ts b/packages/sync/src/types.ts index 03439ecf280319..73d38eb73bddae 100644 --- a/packages/sync/src/types.ts +++ b/packages/sync/src/types.ts @@ -1,11 +1,28 @@ +/** + * External dependencies + */ +import type * as Y from 'yjs'; + +declare global { + interface Window { + __experimentalCollaborativeEditingSecret?: string; + wp: { + ajax: { + settings: { + url: string; + }; + }; + }; + } +} + export type ObjectID = string; export type ObjectType = string; -export type ObjectData = any; -export type CRDTDoc = any; +export type CRDTDoc = Y.Doc; export type ObjectConfig = { - fetch: ( id: ObjectID ) => Promise< ObjectData >; - applyChangesToDoc: ( doc: CRDTDoc, data: any ) => void; + applyChangesToDoc: ( doc: CRDTDoc, data: object ) => void; + fetch: ( id: ObjectID ) => Promise< object >; fromCRDTDoc: ( doc: CRDTDoc ) => any; }; @@ -16,12 +33,11 @@ export type ConnectDoc = ( ) => Promise< () => void >; export type SyncProvider = { - register: ( type: ObjectType, config: ObjectConfig ) => void; bootstrap: ( type: ObjectType, id: ObjectID, - handleChanges: ( data: any ) => void - ) => Promise< CRDTDoc >; - update: ( type: ObjectType, id: ObjectID, data: any ) => void; - discard: ( type: ObjectType, id: ObjectID ) => Promise< CRDTDoc >; + handleChanges: ( data: object ) => void + ) => Promise< void >; + discard: ( type: ObjectType, id: ObjectID ) => Promise< void >; + update: ( type: ObjectType, id: ObjectID, data: object ) => void; }; diff --git a/packages/sync/tsconfig.json b/packages/sync/tsconfig.json index f0a5cb0530d297..53e6a2b663d310 100644 --- a/packages/sync/tsconfig.json +++ b/packages/sync/tsconfig.json @@ -4,5 +4,5 @@ "compilerOptions": { "types": [ "node" ] }, - "references": [ { "path": "../url" } ] + "references": [ { "path": "../hooks" }, { "path": "../url" } ] } From 2879565db8ba58fdac6dc3b810ef955489afd9b1 Mon Sep 17 00:00:00 2001 From: chriszarate Date: Thu, 24 Jul 2025 16:13:21 -0400 Subject: [PATCH 02/54] Promote @wordpress/sync to public package --- packages/dependency-extraction-webpack-plugin/lib/util.js | 1 - tools/webpack/packages.js | 1 - 2 files changed, 2 deletions(-) diff --git a/packages/dependency-extraction-webpack-plugin/lib/util.js b/packages/dependency-extraction-webpack-plugin/lib/util.js index b5c9f9057c2052..11d24cb433026a 100644 --- a/packages/dependency-extraction-webpack-plugin/lib/util.js +++ b/packages/dependency-extraction-webpack-plugin/lib/util.js @@ -8,7 +8,6 @@ const BUNDLED_PACKAGES = [ '@wordpress/dataviews/wp', '@wordpress/icons', '@wordpress/interface', - '@wordpress/sync', '@wordpress/undo-manager', '@wordpress/upload-media', '@wordpress/fields', diff --git a/tools/webpack/packages.js b/tools/webpack/packages.js index c99c25ee0127ce..f847d67e01f455 100644 --- a/tools/webpack/packages.js +++ b/tools/webpack/packages.js @@ -39,7 +39,6 @@ const BUNDLED_PACKAGES = [ '@wordpress/dataviews/wp', '@wordpress/icons', '@wordpress/interface', - '@wordpress/sync', '@wordpress/undo-manager', '@wordpress/upload-media', '@wordpress/fields', From da6ac8e599148dba4436ee09d837925aad12b718 Mon Sep 17 00:00:00 2001 From: chriszarate Date: Thu, 24 Jul 2025 16:52:31 -0400 Subject: [PATCH 03/54] Separate experimental flags for feature and transport --- lib/client-assets.php | 5 ++++- .../class-gutenberg-http-signaling-server.php | 2 +- lib/experiments-page.php | 16 ++++++++++++++-- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/lib/client-assets.php b/lib/client-assets.php index 77e5e79e96c1ab..f3a5165538d9db 100644 --- a/lib/client-assets.php +++ b/lib/client-assets.php @@ -213,7 +213,10 @@ function gutenberg_register_packages_scripts( $scripts ) { break; case 'wp-core-data': - if ( gutenberg_is_experiment_enabled( 'gutenberg-sync-collaboration' ) ) { + if ( + gutenberg_is_experiment_enabled( 'gutenberg-sync-collaboration' ) && + gutenberg_is_experiment_enabled( 'gutenberg-sync-collaboration-webrtc-provider' ) + ) { array_push( $dependencies, 'wp-sync' ); } break; diff --git a/lib/experimental/sync/class-gutenberg-http-signaling-server.php b/lib/experimental/sync/class-gutenberg-http-signaling-server.php index 3448e82a4c31d0..3570599e299cd3 100644 --- a/lib/experimental/sync/class-gutenberg-http-signaling-server.php +++ b/lib/experimental/sync/class-gutenberg-http-signaling-server.php @@ -19,7 +19,7 @@ class Gutenberg_HTTP_Signaling_Server { */ public static function init() { $gutenberg_experiments = get_option( 'gutenberg-experiments' ); - if ( ! $gutenberg_experiments || ! array_key_exists( 'gutenberg-sync-collaboration', $gutenberg_experiments ) ) { + if ( ! $gutenberg_experiments || ! array_key_exists( 'gutenberg-sync-collaboration-webrtc-provider', $gutenberg_experiments ) ) { return; } add_action( 'wp_ajax_gutenberg_signaling_server', array( __CLASS__, 'do_wp_ajax_action' ) ); diff --git a/lib/experiments-page.php b/lib/experiments-page.php index a2032eb99abdc5..f990d4699d7a33 100644 --- a/lib/experiments-page.php +++ b/lib/experiments-page.php @@ -117,16 +117,28 @@ function gutenberg_initialize_experiments_settings() { add_settings_field( 'gutenberg-sync-collaboration', - __( 'Collaboration: add real time editing', 'gutenberg' ), + __( 'Collaboration: enable real-time collaboration', 'gutenberg' ), 'gutenberg_display_experiment_field', 'gutenberg-experiments', 'gutenberg_experiments_section', array( - 'label' => __( 'Enables live collaboration and offline persistence between peers.', 'gutenberg' ), + 'label' => __( 'Enables real-time collaboration between peers (requires provider).', 'gutenberg' ), 'id' => 'gutenberg-sync-collaboration', ) ); + add_settings_field( + 'gutenberg-sync-collaboration-webrtc-provider', + __( 'Collaboration: WebRTC provider', 'gutenberg' ), + 'gutenberg_display_experiment_field', + 'gutenberg-experiments', + 'gutenberg_experiments_section', + array( + 'label' => __( 'Enables the WebRTC provider for real-time collaboration.', 'gutenberg' ), + 'id' => 'gutenberg-sync-collaboration-webrtc-provider', + ) + ); + add_settings_field( 'gutenberg-color-randomizer', __( 'Color randomizer', 'gutenberg' ), From b354743d5e40dfc06c31b66e1af4460a70c2abb9 Mon Sep 17 00:00:00 2001 From: chriszarate Date: Fri, 25 Jul 2025 16:00:31 -0400 Subject: [PATCH 04/54] Restore `register` property, for now. --- packages/sync/src/types.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/sync/src/types.ts b/packages/sync/src/types.ts index 73d38eb73bddae..c141b3e17d6eb8 100644 --- a/packages/sync/src/types.ts +++ b/packages/sync/src/types.ts @@ -39,5 +39,6 @@ export type SyncProvider = { handleChanges: ( data: object ) => void ) => Promise< void >; discard: ( type: ObjectType, id: ObjectID ) => Promise< void >; + register: ( type: ObjectType, config: ObjectConfig ) => void; update: ( type: ObjectType, id: ObjectID, data: object ) => void; }; From 019fd346a8bcb9f9944772a105bf11c4985ad14e Mon Sep 17 00:00:00 2001 From: chriszarate Date: Fri, 25 Jul 2025 16:20:03 -0400 Subject: [PATCH 05/54] Fix unit test ... but dangerously, as I don't fully understand it! --- phpunit/script-dependencies-test.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/phpunit/script-dependencies-test.php b/phpunit/script-dependencies-test.php index 75064dd49cef89..77c229874b6239 100644 --- a/phpunit/script-dependencies-test.php +++ b/phpunit/script-dependencies-test.php @@ -42,7 +42,7 @@ public function test_polyfill_dependents() { 'wp-block-library', 'wp-blocks', 'wp-edit-site', - 'wp-core-data', + 'wp-sync', 'wp-editor', 'wp-router', 'wp-url', From 4144f8cc85ee6237f7bb28699ecf817455b97190 Mon Sep 17 00:00:00 2001 From: chriszarate Date: Mon, 28 Jul 2025 13:47:42 -0400 Subject: [PATCH 06/54] Remove conditional loading of @wordpress/sync --- lib/client-assets.php | 9 ----- packages/core-data/src/sync.ts | 19 ++++++--- .../lib/util.js | 1 + packages/sync/README.md | 8 ++++ packages/sync/src/index.ts | 40 ++++++++----------- phpunit/script-dependencies-test.php | 2 +- tools/webpack/packages.js | 1 + 7 files changed, 41 insertions(+), 39 deletions(-) diff --git a/lib/client-assets.php b/lib/client-assets.php index f3a5165538d9db..e6f64106cbb323 100644 --- a/lib/client-assets.php +++ b/lib/client-assets.php @@ -212,15 +212,6 @@ function gutenberg_register_packages_scripts( $scripts ) { } break; - case 'wp-core-data': - if ( - gutenberg_is_experiment_enabled( 'gutenberg-sync-collaboration' ) && - gutenberg_is_experiment_enabled( 'gutenberg-sync-collaboration-webrtc-provider' ) - ) { - array_push( $dependencies, 'wp-sync' ); - } - break; - case 'wp-edit-post': array_push( $dependencies, 'media-models', 'media-views', 'postbox' ); break; diff --git a/packages/core-data/src/sync.ts b/packages/core-data/src/sync.ts index f262a8508b361a..0884c79cb217a5 100644 --- a/packages/core-data/src/sync.ts +++ b/packages/core-data/src/sync.ts @@ -2,14 +2,15 @@ * WordPress dependencies */ import { applyFilters } from '@wordpress/hooks'; - -/** - * IMPORTANT: Ensure that we only import types from `@wordpress/sync` and not - * code. The code is loaded only behind an experimental flag, which allows - * other plugins to load their own sync provider. - */ +import { getWebRTCSyncProvider } from '@wordpress/sync'; import type { SyncProvider } from '@wordpress/sync'; +declare global { + interface Window { + __experimentalEnableSync?: boolean; + } +} + let syncProvider: SyncProvider; /** @@ -39,5 +40,11 @@ export function getSyncProvider(): SyncProvider { null ) as SyncProvider | null ) ?? fallbackNoOpSyncProvider; + // If the filter does not produce a provider and the experimental flag is set, + // get the WebRTC sync provider. + if ( ! syncProvider && window.__experimentalEnableSync ) { + syncProvider = getWebRTCSyncProvider(); + } + return syncProvider; } diff --git a/packages/dependency-extraction-webpack-plugin/lib/util.js b/packages/dependency-extraction-webpack-plugin/lib/util.js index 11d24cb433026a..b5c9f9057c2052 100644 --- a/packages/dependency-extraction-webpack-plugin/lib/util.js +++ b/packages/dependency-extraction-webpack-plugin/lib/util.js @@ -8,6 +8,7 @@ const BUNDLED_PACKAGES = [ '@wordpress/dataviews/wp', '@wordpress/icons', '@wordpress/interface', + '@wordpress/sync', '@wordpress/undo-manager', '@wordpress/upload-media', '@wordpress/fields', diff --git a/packages/sync/README.md b/packages/sync/README.md index 6b8f984463d5b5..ac18815fcbbea1 100644 --- a/packages/sync/README.md +++ b/packages/sync/README.md @@ -53,6 +53,14 @@ _Returns_ - `ConnectDoc`: Promise that resolves when the connection is established. +### getWebRTCSyncProvider + +Returns a WebRTC sync provider. This is the curent default sync provider. + +_Returns_ + +- `SyncProvider`: The WebRTC sync provider. + ## Contributing to this package diff --git a/packages/sync/src/index.ts b/packages/sync/src/index.ts index 3b6a625a32300e..8179d787ef86cc 100644 --- a/packages/sync/src/index.ts +++ b/packages/sync/src/index.ts @@ -1,7 +1,6 @@ /** * WordPress dependencies */ -import { addFilter } from '@wordpress/hooks'; /** * Internal dependencies @@ -16,25 +15,20 @@ export { createWebRTCConnection } from './create-webrtc-connection'; export { createSyncProvider } from './provider'; export * from './types'; -addFilter( - 'core.getSyncProvider', - 'wordpress-sync-webrtc/get-sync-provider', - ( provider: SyncProvider | null ): SyncProvider => { - // Do not override an already defined sync provider. - if ( provider ) { - return provider; - } - - return createSyncProvider( - connectIndexDb, - createWebRTCConnection( { - password: window?.__experimentalCollaborativeEditingSecret, - signaling: [ - //'ws://localhost:4444', - window?.wp?.ajax?.settings?.url, - ], - } ) - ); - }, - 10 -); +/** + * Returns a WebRTC sync provider. This is the curent default sync provider. + * + * @return {SyncProvider} The WebRTC sync provider. + */ +export function getWebRTCSyncProvider(): SyncProvider { + return createSyncProvider( + connectIndexDb, + createWebRTCConnection( { + password: window?.__experimentalCollaborativeEditingSecret, + signaling: [ + //'ws://localhost:4444', + window?.wp?.ajax?.settings?.url, + ], + } ) + ); +} diff --git a/phpunit/script-dependencies-test.php b/phpunit/script-dependencies-test.php index 77c229874b6239..75064dd49cef89 100644 --- a/phpunit/script-dependencies-test.php +++ b/phpunit/script-dependencies-test.php @@ -42,7 +42,7 @@ public function test_polyfill_dependents() { 'wp-block-library', 'wp-blocks', 'wp-edit-site', - 'wp-sync', + 'wp-core-data', 'wp-editor', 'wp-router', 'wp-url', diff --git a/tools/webpack/packages.js b/tools/webpack/packages.js index f847d67e01f455..c99c25ee0127ce 100644 --- a/tools/webpack/packages.js +++ b/tools/webpack/packages.js @@ -39,6 +39,7 @@ const BUNDLED_PACKAGES = [ '@wordpress/dataviews/wp', '@wordpress/icons', '@wordpress/interface', + '@wordpress/sync', '@wordpress/undo-manager', '@wordpress/upload-media', '@wordpress/fields', From b91ef4b4ba0bd29d2ff922fc1a39b5c9b3d6bc82 Mon Sep 17 00:00:00 2001 From: chriszarate Date: Mon, 28 Jul 2025 13:57:54 -0400 Subject: [PATCH 07/54] Remove @wordpress/hooks dependency --- packages/sync/package.json | 1 - packages/sync/tsconfig.json | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/sync/package.json b/packages/sync/package.json index af982be222d900..772979b2907c37 100644 --- a/packages/sync/package.json +++ b/packages/sync/package.json @@ -31,7 +31,6 @@ "dependencies": { "@babel/runtime": "7.25.7", "@types/simple-peer": "^9.11.5", - "@wordpress/hooks": "file:../hooks", "@wordpress/url": "file:../url", "import-locals": "^2.0.0", "lib0": "^0.2.42", diff --git a/packages/sync/tsconfig.json b/packages/sync/tsconfig.json index 53e6a2b663d310..f0a5cb0530d297 100644 --- a/packages/sync/tsconfig.json +++ b/packages/sync/tsconfig.json @@ -4,5 +4,5 @@ "compilerOptions": { "types": [ "node" ] }, - "references": [ { "path": "../hooks" }, { "path": "../url" } ] + "references": [ { "path": "../url" } ] } From 8d623000c53c76911df940c480ad4f09e08278f2 Mon Sep 17 00:00:00 2001 From: chriszarate Date: Mon, 28 Jul 2025 13:58:09 -0400 Subject: [PATCH 08/54] Remove new experimental flag --- .../sync/class-gutenberg-http-signaling-server.php | 2 +- lib/experiments-page.php | 14 +------------- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/lib/experimental/sync/class-gutenberg-http-signaling-server.php b/lib/experimental/sync/class-gutenberg-http-signaling-server.php index 3570599e299cd3..3448e82a4c31d0 100644 --- a/lib/experimental/sync/class-gutenberg-http-signaling-server.php +++ b/lib/experimental/sync/class-gutenberg-http-signaling-server.php @@ -19,7 +19,7 @@ class Gutenberg_HTTP_Signaling_Server { */ public static function init() { $gutenberg_experiments = get_option( 'gutenberg-experiments' ); - if ( ! $gutenberg_experiments || ! array_key_exists( 'gutenberg-sync-collaboration-webrtc-provider', $gutenberg_experiments ) ) { + if ( ! $gutenberg_experiments || ! array_key_exists( 'gutenberg-sync-collaboration', $gutenberg_experiments ) ) { return; } add_action( 'wp_ajax_gutenberg_signaling_server', array( __CLASS__, 'do_wp_ajax_action' ) ); diff --git a/lib/experiments-page.php b/lib/experiments-page.php index f990d4699d7a33..0ef9fff443432a 100644 --- a/lib/experiments-page.php +++ b/lib/experiments-page.php @@ -122,23 +122,11 @@ function gutenberg_initialize_experiments_settings() { 'gutenberg-experiments', 'gutenberg_experiments_section', array( - 'label' => __( 'Enables real-time collaboration between peers (requires provider).', 'gutenberg' ), + 'label' => __( 'Enables real-time collaboration between peers.', 'gutenberg' ), 'id' => 'gutenberg-sync-collaboration', ) ); - add_settings_field( - 'gutenberg-sync-collaboration-webrtc-provider', - __( 'Collaboration: WebRTC provider', 'gutenberg' ), - 'gutenberg_display_experiment_field', - 'gutenberg-experiments', - 'gutenberg_experiments_section', - array( - 'label' => __( 'Enables the WebRTC provider for real-time collaboration.', 'gutenberg' ), - 'id' => 'gutenberg-sync-collaboration-webrtc-provider', - ) - ); - add_settings_field( 'gutenberg-color-randomizer', __( 'Color randomizer', 'gutenberg' ), From 2cfe35c53826c214297c68f3794becc986e729b8 Mon Sep 17 00:00:00 2001 From: chriszarate Date: Mon, 28 Jul 2025 14:09:26 -0400 Subject: [PATCH 09/54] Correctly fall back to no-op sync provider --- packages/core-data/src/sync.ts | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/packages/core-data/src/sync.ts b/packages/core-data/src/sync.ts index 0884c79cb217a5..a9295508c1ecbc 100644 --- a/packages/core-data/src/sync.ts +++ b/packages/core-data/src/sync.ts @@ -11,7 +11,7 @@ declare global { } } -let syncProvider: SyncProvider; +let syncProvider: SyncProvider | null = null; /** * Returns the current sync provider, filterable by external code. @@ -34,11 +34,10 @@ export function getSyncProvider(): SyncProvider { update: () => {}, }; - syncProvider = - ( applyFilters( - 'core.getSyncProvider', - null - ) as SyncProvider | null ) ?? fallbackNoOpSyncProvider; + syncProvider = applyFilters( + 'core.getSyncProvider', + null + ) as SyncProvider | null; // If the filter does not produce a provider and the experimental flag is set, // get the WebRTC sync provider. @@ -46,5 +45,10 @@ export function getSyncProvider(): SyncProvider { syncProvider = getWebRTCSyncProvider(); } + // If no sync provider is set, use the fallback no-op sync provider. + if ( ! syncProvider ) { + syncProvider = fallbackNoOpSyncProvider; + } + return syncProvider; } From 3865ac97c879f415dc8e62411e5a3d03d2456ca6 Mon Sep 17 00:00:00 2001 From: chriszarate Date: Mon, 28 Jul 2025 14:09:51 -0400 Subject: [PATCH 10/54] Remove dependency from base package-lock --- package-lock.json | 1 - 1 file changed, 1 deletion(-) diff --git a/package-lock.json b/package-lock.json index 574cb822fa8784..b924aa67befad9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -52308,7 +52308,6 @@ "dependencies": { "@babel/runtime": "7.25.7", "@types/simple-peer": "^9.11.5", - "@wordpress/hooks": "file:../hooks", "@wordpress/url": "file:../url", "import-locals": "^2.0.0", "lib0": "^0.2.42", From 571c9519d80b59caefb4877aeee2473f23360c0a Mon Sep 17 00:00:00 2001 From: chriszarate Date: Tue, 29 Jul 2025 11:16:50 -0400 Subject: [PATCH 11/54] Add Yjs-aware entity config --- package-lock.json | 32 +- packages/core-data/package.json | 1 + packages/core-data/src/actions.js | 56 +- packages/core-data/src/entities.js | 537 +++++++++++++++--- packages/core-data/src/resolvers.js | 182 +++--- packages/core-data/src/sync.ts | 4 +- packages/core-data/tsconfig.json | 3 +- .../lib/util.js | 1 - packages/sync/CODE.md | 2 +- packages/sync/package.json | 11 +- packages/sync/src/connect-indexdb.js | 11 +- packages/sync/src/create-webrtc-connection.ts | 5 +- packages/sync/src/index.ts | 14 + packages/sync/src/provider.js | 128 ----- packages/sync/src/provider.ts | 258 +++++++++ packages/sync/src/types.ts | 77 ++- tools/webpack/packages.js | 1 - 17 files changed, 924 insertions(+), 399 deletions(-) delete mode 100644 packages/sync/src/provider.js create mode 100644 packages/sync/src/provider.ts diff --git a/package-lock.json b/package-lock.json index b924aa67befad9..ba91ec4675d642 100644 --- a/package-lock.json +++ b/package-lock.json @@ -27697,11 +27697,6 @@ "node": ">=8" } }, - "node_modules/import-locals": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/import-locals/-/import-locals-2.0.0.tgz", - "integrity": "sha512-1/bPE89IZhyf7dr5Pkz7b4UyVXy5pEt7PTEfye15UEn3AK8+2zwcDCfKk9Pwun4ltfhOSszOrReSsFcDKw/yoA==" - }, "node_modules/import-meta-resolve": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/import-meta-resolve/-/import-meta-resolve-3.1.1.tgz", @@ -31285,13 +31280,15 @@ } }, "node_modules/lib0": { - "version": "0.2.79", - "resolved": "https://registry.npmjs.org/lib0/-/lib0-0.2.79.tgz", - "integrity": "sha512-fIdPbxzMVq10wt3ou1lp3/f9n5ciHZ6t+P1vyGy3XXr018AntTYM4eg24sNFcNq8SYDQwmhhoGdS58IlYBzfBw==", + "version": "0.2.114", + "resolved": "https://registry.npmjs.org/lib0/-/lib0-0.2.114.tgz", + "integrity": "sha512-gcxmNFzA4hv8UYi8j43uPlQ7CGcyMJ2KQb5kZASw6SnAKAf10hK12i2fjrS3Cl/ugZa5Ui6WwIu1/6MIXiHttQ==", + "license": "MIT", "dependencies": { "isomorphic.js": "^0.2.4" }, "bin": { + "0ecdsa-generate-keypair": "bin/0ecdsa-generate-keypair.js", "0gentesthtml": "bin/gentesthtml.js", "0serve": "bin/0serve.js" }, @@ -48816,9 +48813,10 @@ } }, "node_modules/yjs": { - "version": "13.6.7", - "resolved": "https://registry.npmjs.org/yjs/-/yjs-13.6.7.tgz", - "integrity": "sha512-mCZTh4kjvUS2DnaktsYN6wLH3WZCJBLqrTdkWh1bIDpA/sB/GNFaLA/dyVJj2Hc7KwONuuoC/vWe9bwBBosZLQ==", + "version": "13.6.6", + "resolved": "https://registry.npmjs.org/yjs/-/yjs-13.6.6.tgz", + "integrity": "sha512-VTvezMeMuOra9jKG1Ym5XuQ2H4xXOubIIIupv/B5oygasa9IqDE7Ufv93QTSe9uz69J5VZGMQb2WTEmJv4kJFQ==", + "license": "MIT", "dependencies": { "lib0": "^0.2.74" }, @@ -50373,6 +50371,7 @@ "change-case": "^4.1.2", "equivalent-key-map": "^0.2.2", "fast-deep-equal": "^3.1.3", + "lib0": "^0.2.99", "memize": "^2.1.0", "uuid": "^9.0.1" }, @@ -52306,16 +52305,15 @@ "version": "1.27.0", "license": "GPL-2.0-or-later", "dependencies": { - "@babel/runtime": "7.25.7", "@types/simple-peer": "^9.11.5", + "@wordpress/hooks": "file:../hooks", "@wordpress/url": "file:../url", - "import-locals": "^2.0.0", - "lib0": "^0.2.42", + "lib0": "^0.2.99", "simple-peer": "^9.11.0", - "y-indexeddb": "~9.0.11", + "y-indexeddb": "^9.0.11", "y-protocols": "^1.0.5", - "y-webrtc": "~10.2.5", - "yjs": "~13.6.6" + "y-webrtc": "^10.2.5", + "yjs": "13.6.6" }, "engines": { "node": ">=18.12.0", diff --git a/packages/core-data/package.json b/packages/core-data/package.json index cd0593c5f27052..313fc77a847dea 100644 --- a/packages/core-data/package.json +++ b/packages/core-data/package.json @@ -53,6 +53,7 @@ "change-case": "^4.1.2", "equivalent-key-map": "^0.2.2", "fast-deep-equal": "^3.1.3", + "lib0": "^0.2.99", "memize": "^2.1.0", "uuid": "^9.0.1" }, diff --git a/packages/core-data/src/actions.js b/packages/core-data/src/actions.js index 275cfccdb7823a..d01a11bff129f6 100644 --- a/packages/core-data/src/actions.js +++ b/packages/core-data/src/actions.js @@ -397,39 +397,37 @@ export const editEntityRecord = }; if ( window.__experimentalEnableSync && entityConfig.syncConfig ) { if ( globalThis.IS_GUTENBERG_PLUGIN ) { - const objectId = entityConfig.getSyncObjectId( recordId ); + // @todo this always updates the Yjs doc, which is undesirable, probably we can read the yjs + // content from the comment tag here getSyncProvider().update( - entityConfig.syncObjectType + '--edit', - objectId, - edit.edits + entityConfig.syncConfig.objectType, + record, + edit.edits, + 'gutenberg' ); } - } else { - if ( ! options.undoIgnore ) { - select.getUndoManager().addRecord( - [ - { - id: { kind, name, recordId }, - changes: Object.keys( edits ).reduce( - ( acc, key ) => { - acc[ key ] = { - from: editedRecord[ key ], - to: edits[ key ], - }; - return acc; - }, - {} - ), - }, - ], - options.isCached - ); - } - dispatch( { - type: 'EDIT_ENTITY_RECORD', - ...edit, - } ); } + if ( ! options.undoIgnore ) { + select.getUndoManager().addRecord( + [ + { + id: { kind, name, recordId }, + changes: Object.keys( edits ).reduce( ( acc, key ) => { + acc[ key ] = { + from: editedRecord[ key ], + to: edits[ key ], + }; + return acc; + }, {} ), + }, + ], + options.isCached + ); + } + dispatch( { + type: 'EDIT_ENTITY_RECORD', + ...edit, + } ); }; /** diff --git a/packages/core-data/src/entities.js b/packages/core-data/src/entities.js index e3fa8b56b73447..85f381ef5ef7fe 100644 --- a/packages/core-data/src/entities.js +++ b/packages/core-data/src/entities.js @@ -2,17 +2,106 @@ * External dependencies */ import { capitalCase, pascalCase } from 'change-case'; - +import { v4 as uuidv4 } from 'uuid'; +import * as string from 'lib0/string'; +import * as sha256 from 'lib0/hash/sha256'; /** * WordPress dependencies */ import apiFetch from '@wordpress/api-fetch'; import { __ } from '@wordpress/i18n'; import { RichTextData } from '@wordpress/rich-text'; +import { parse } from '@wordpress/blocks'; +import { Y } from '@wordpress/sync'; +import * as math from 'lib0/math'; +import * as fun from 'lib0/function'; + +/** + * Internal dependencies + */ +import { getSyncProvider } from './sync'; export const DEFAULT_ENTITY_KEY = 'id'; const POST_RAW_ATTRIBUTES = [ 'title', 'excerpt', 'content' ]; +// @todo refactor `applyChangesToDoc` implementations this to have less repetition (there are +// multiple similar implementations) + +/** + * Similar to `parse`, but only reads the Yjs document if available. + * @param {string} postType + * @param {string} content + */ +export function parseContentYdoc( postType, content ) { + const newClientId = new Uint32Array( + sha256.digest( string.encodeUtf8( content ) ).buffer + ); + const syncProvider = getSyncProvider(); + + // It is important that this is a fresh document - don't use the document from the sync package! + const ydoc = new Y.Doc( { meta: new Map() } ); + + const knownUpdateGuids = new Set(); + ydoc.meta.set( 'knownRemoteUpdates', knownUpdateGuids ); + // Changing the Yjs clientid may lead to very weird bugs if done incorrectly. + // Please handle the following code-portion with great care! + const prevClientId = ydoc.clientID; + ydoc.clientID = newClientId; + const prevClock = ( ydoc.store.clients.get( newClientId ) || [ + { id: { clock: 0 } }, + ] )[ 0 ].id.clock; + const blocks = parse( content ); + syncProvider.configs.get( postType ).applyChangesToDoc( ydoc, { + blocks, + } ); + ydoc.clientID = prevClientId; + const newClock = ( ydoc.store.clients.get( newClientId ) ?? [ + { id: { clock: 0 } }, + ] )[ 0 ].id.clock; + if ( prevClock !== newClock ) { + // eslint-disable-next-line no-console + console.info( + '[Yjs Collab] Yjs document was updated to reflect changes to the HTML document.' + ); + } + return ydoc; +} + +// only sync what is necessary! +const filteredAttributes = new Set( [ + 'content', + 'selection', + 'excerpt', + 'date', + 'date_gmt', + 'format', + 'generated_slug', + 'link', + 'meta', + 'modified', + 'modified_gmt', + 'slug', + 'status', + 'sticky', + 'tags', + 'template', + '_links', + 'id', + 'password', + 'featured_media', +] ); + +/** + * @param {Y.Doc} ydoc + */ +const defaultYdocTransformer = ( ydoc ) => { + const json = ydoc.getMap( 'document' ).toJSON(); + if ( json.title?.raw ) { + json.title = json.title.raw; + } + return json; +}; + export const rootEntitiesConfig = [ { label: __( 'Base' ), @@ -41,23 +130,43 @@ export const rootEntitiesConfig = [ // The property is maintained for backward compatibility. plural: '__unstableBases', syncConfig: { - fetch: async () => { - return apiFetch( { path: '/' } ); - }, applyChangesToDoc: ( doc, changes ) => { - const document = doc.getMap( 'document' ); - Object.entries( changes ).forEach( ( [ key, value ] ) => { - if ( document.get( key ) !== value ) { - document.set( key, value ); - } - } ); - }, - fromCRDTDoc: ( doc ) => { - return doc.getMap( 'document' ).toJSON(); + const content = changes.content?.raw || changes.content; + const parsedYdoc = + typeof content === 'string' + ? parseContentYdoc( 'root/base', content ) + : null; // Note: always use the same 'postType' as this object's config.syncObjectType + if ( parsedYdoc !== null ) { + // parse content which contains a ydoc, and apply it to the current ydoc. The rest of the attributes can be ignored. + Y.transact( + doc, + () => { + // apply remote changes + Y.applyUpdate( + doc, + Y.encodeStateAsUpdate( parsedYdoc ) + ); + }, + 'applyChangesToDoc', + false + ); + } else { + // local changes happened. Apply the differences to the ydoc + const ycontent = doc.getMap( 'document' ); + Object.entries( changes ).forEach( ( [ key, value ] ) => { + if ( + ! filteredAttributes.has( key ) && + ! fun.equalityDeep( ycontent.get( key ), value ) + ) { + ycontent.set( key, value ); + } + } ); + } }, + fromCRDTDoc: defaultYdocTransformer, + getObjectId: () => 'index', + objectType: 'root/base', }, - syncObjectType: 'root/base', - getSyncObjectId: () => 'index', }, { label: __( 'Post Type' ), @@ -68,25 +177,43 @@ export const rootEntitiesConfig = [ baseURLParams: { context: 'edit' }, plural: 'postTypes', syncConfig: { - fetch: async ( id ) => { - return apiFetch( { - path: `/wp/v2/types/${ id }?context=edit`, - } ); - }, - applyChangesToDoc: ( doc, changes ) => { - const document = doc.getMap( 'document' ); - Object.entries( changes ).forEach( ( [ key, value ] ) => { - if ( document.get( key ) !== value ) { - document.set( key, value ); - } - } ); - }, - fromCRDTDoc: ( doc ) => { - return doc.getMap( 'document' ).toJSON(); + applyChangesToDoc: ( ydoc, changes ) => { + const content = changes.content?.raw || changes.content; + const parsedYdoc = + typeof content === 'string' + ? parseContentYdoc( 'root/postType', content ) + : null; // Note: always use the same 'postType' as this object's config.syncObjectType + if ( parsedYdoc !== null ) { + // parse content which contains a ydoc, and apply it to the current ydoc. The rest of the attributes can be ignored. + Y.transact( + ydoc, + () => { + // apply remote changes + Y.applyUpdate( + ydoc, + Y.encodeStateAsUpdate( parsedYdoc ) + ); + }, + 'applyChangesToDoc', + false + ); + } else { + // local changes happened. Apply the differences to the ydoc + const ycontent = ydoc.getMap( 'document' ); + Object.entries( changes ).forEach( ( [ key, value ] ) => { + if ( + ! filteredAttributes.has( key ) && + ! fun.equalityDeep( ycontent.get( key ), value ) + ) { + ycontent.set( key, value ); + } + } ); + } }, + fromCRDTDoc: defaultYdocTransformer, + getObjectId: ( { id } ) => id, + objectType: 'root/postType', }, - syncObjectType: 'root/postType', - getSyncObjectId: ( id ) => id, }, { name: 'media', @@ -275,6 +402,9 @@ function makeBlockAttributesSerializable( attributes ) { function makeBlocksSerializable( blocks ) { return blocks.map( ( block ) => { const { innerBlocks, attributes, ...rest } = block; + delete rest.validationIssues; + delete rest.originalContent; + // delete rest.isValid return { ...rest, attributes: makeBlockAttributesSerializable( attributes ), @@ -318,39 +448,274 @@ async function loadPostTypeEntities() { __unstablePrePersist: isTemplate ? undefined : prePersistPostType, __unstable_rest_base: postType.rest_base, syncConfig: { - fetch: async ( id ) => { - return apiFetch( { - path: `/${ namespace }/${ postType.rest_base }/${ id }?context=edit`, - } ); - }, - applyChangesToDoc: ( doc, changes ) => { - const document = doc.getMap( 'document' ); - - Object.entries( changes ).forEach( ( [ key, value ] ) => { - if ( typeof value !== 'function' ) { - if ( key === 'blocks' ) { - if ( ! serialisableBlocksCache.has( value ) ) { - serialisableBlocksCache.set( - value, - makeBlocksSerializable( value ) - ); + /** + * @param {Y.Doc} ydoc + * @param {any} changes + */ + applyChangesToDoc: ( ydoc, changes ) => { + const content = changes.content?.raw || changes.content; + const parsedYdoc = + typeof content === 'string' + ? parseContentYdoc( + 'postType/' + postType.name, + content + ) + : null; // Note: always use the same 'postType' as this object's config.syncObjectType + if ( parsedYdoc !== null ) { + // parse content which contains a ydoc, and apply it to the current ydoc. The rest of the attributes can be ignored. + Y.transact( + ydoc, + () => { + // apply remote changes + Y.applyUpdate( + ydoc, + Y.encodeStateAsUpdate( parsedYdoc ) + ); + }, + 'applyChangesToDoc', + false + ); + } else { + // local changes happened. Apply the differences to the ydoc + const ycontent = ydoc.getMap( 'document' ); + ydoc.transact( () => { + Object.entries( changes ).forEach( + ( [ key, value ] ) => { + if ( typeof value !== 'function' ) { + if ( key === 'blocks' ) { + if ( + ! serialisableBlocksCache.has( + value + ) + ) { + serialisableBlocksCache.set( + value, + makeBlocksSerializable( + value + ) + ); + } + const blocks = + serialisableBlocksCache.get( + value + ); + // This is a rudimentary diff implementation similar to the y-prosemirror diffing + // approach. + // A better implementation would also diff the textual content and represent it + // using a Y.Text type. + // However, at this time it makes more sense to keep this algorithm generic to + // support all kinds of block types. + // Ideally, we ensure that block data structure have a consistent data format. + // E.g.: + // - textual content (using rich-text formatting?) may always be stored under `block.text` + // - local information that shouldn't be shared (e.g. clientId or isDragging) is stored under `block.private` + if ( + ! ycontent.has( key ) || + ycontent.get( key ) instanceof + Array + ) { + // @todo remove the array check + ycontent.set( + key, + new Y.Array() + ); + } + /** + * @type {Y.Array>} + */ + const yblocks = ycontent.get( key ); + const numOfCommonEntries = math.min( + blocks.length, + yblocks.length + ); + let left = 0; + let right = 0; + /** + * @param {any} gblock + * @param {Y.Map} yblock + */ + const blocksEqual = ( + gblock, + yblock + ) => { + if ( yblock.toJSON ) { + yblock = yblock.toJSON(); + } + // we must not sync clientId, as this can't be generated consistenctly and + // hence will lead to merge conflicts. + const overwrites = { + innerBlocks: null, + clientId: null, + }; + const res = fun.equalityDeep( + Object.assign( + {}, + gblock, + overwrites + ), + Object.assign( + {}, + yblock, + overwrites + ) + ); + const inners = + gblock.innerBlocks || []; + const yinners = + yblock.innerBlocks || []; + return ( + res && + inners.length === + yinners.length && + inners.every( + ( block, i ) => + blocksEqual( + block, + yinners[ i ] + ) + ) + ); + }; + // skip equal blocks from left + for ( + ; + left < numOfCommonEntries && + blocksEqual( + blocks[ left ], + yblocks.get( left ) + ); + left++ + ) { + /* nop */ + } + // skip equal blocks from right + for ( + ; + right < + numOfCommonEntries - left && + blocksEqual( + blocks[ + blocks.length - + right - + 1 + ], + yblocks.get( + yblocks.length - + right - + 1 + ) + ); + right++ + ) { + /* nop */ + } + const numOfUpdatesNeeded = + numOfCommonEntries - + left - + right; + const numOfInsertionsNeeded = + math.max( + 0, + blocks.length - + yblocks.length + ); + const numOfDeletionsNeeded = + math.max( + 0, + yblocks.length - + blocks.length + ); + // updates + for ( + let i = 0; + i < numOfUpdatesNeeded; + i++, left++ + ) { + const block = blocks[ left ]; + const yblock = + yblocks.get( left ); + Object.entries( block ).forEach( + ( [ k, v ] ) => { + if ( + ! fun.equalityDeep( + block[ k ], + yblock.get( k ) + ) + ) { + yblock.set( k, v ); + } + } + ); + yblock.forEach( ( _v, k ) => { + if ( + ! block.hasOwnProperty( + k + ) + ) { + yblock.delete( k ); + } + } ); + } + // deletes + yblocks.delete( + left, + numOfDeletionsNeeded + ); + // inserts + for ( + let i = 0; + i < numOfInsertionsNeeded; + i++, left++ + ) { + yblocks.insert( left, [ + new Y.Map( + Object.entries( + blocks[ left ] + ) + ), + ] ); + } + const knownClientIds = new Set(); + // remove duplicate clientids + for ( + let j = 0; + j < yblocks.length; + j++ + ) { + const yblock = yblocks.get( j ); + if ( + knownClientIds.has( + yblock.get( 'clientId' ) + ) + ) { + yblock.set( + 'clientId', + uuidv4() + ); + } + knownClientIds.add( + yblock.get( 'clientId' ) + ); + } + } else if ( + ! filteredAttributes.has( key ) && + ! fun.equalityDeep( + ycontent.get( key ), + value + ) + ) { + ycontent.set( key, value ); + } + } } - - value = serialisableBlocksCache.get( value ); - } - - if ( document.get( key ) !== value ) { - document.set( key, value ); - } - } - } ); - }, - fromCRDTDoc: ( doc ) => { - return doc.getMap( 'document' ).toJSON(); + ); + }, 'gutenberg' ); + } }, + fromCRDTDoc: defaultYdocTransformer, + getObjectId: ( { id } ) => id, + objectType: 'postType/' + postType.name, }, - syncObjectType: 'postType/' + postType.name, - getSyncObjectId: ( id ) => id, supportsPagination: true, getRevisionsUrl: ( parentId, revisionId ) => `/${ namespace }/${ @@ -397,23 +762,43 @@ async function loadSiteEntity() { kind: 'root', baseURL: '/wp/v2/settings', syncConfig: { - fetch: async () => { - return apiFetch( { path: '/wp/v2/settings' } ); - }, applyChangesToDoc: ( doc, changes ) => { - const document = doc.getMap( 'document' ); - Object.entries( changes ).forEach( ( [ key, value ] ) => { - if ( document.get( key ) !== value ) { - document.set( key, value ); - } - } ); - }, - fromCRDTDoc: ( doc ) => { - return doc.getMap( 'document' ).toJSON(); + const content = changes.content?.raw || changes.content; + const parsedYdoc = + typeof content === 'string' + ? parseContentYdoc( 'root/site', content ) + : null; // Note: always use the same 'postType' as this object's config.syncObjectType + if ( parsedYdoc !== null ) { + // parse content which contains a ydoc, and apply it to the current ydoc. The rest of the attributes can be ignored. + Y.transact( + doc, + () => { + // apply remote changes + Y.applyUpdate( + doc, + Y.encodeStateAsUpdate( parsedYdoc ) + ); + }, + 'applyChangesToDoc', + false + ); + } else { + // local changes happened. Apply the differences to the ydoc + const ycontent = doc.getMap( 'document' ); + Object.entries( changes ).forEach( ( [ key, value ] ) => { + if ( + ! filteredAttributes.has( key ) && + ! fun.equalityDeep( ycontent.get( key ), value ) + ) { + ycontent.set( key, value ); + } + } ); + } }, + fromCRDTDoc: defaultYdocTransformer, + getObjectId: () => 'index', + objectType: 'root/site', }, - syncObjectType: 'root/site', - getSyncObjectId: () => 'index', meta: {}, }; diff --git a/packages/core-data/src/resolvers.js b/packages/core-data/src/resolvers.js index 2da5778580d63a..a597284807fbb3 100644 --- a/packages/core-data/src/resolvers.js +++ b/packages/core-data/src/resolvers.js @@ -73,49 +73,97 @@ export const getEntityRecord = if ( ! entityConfig ) { return; } - const lock = await dispatch.__unstableAcquireStoreLock( STORE_NAME, [ 'entities', 'records', kind, name, key ], { exclusive: false } ); - try { // Entity supports configs, - // use the sync algorithm instead of the old fetch behavior. + if ( query !== undefined && query._fields ) { + // @todo how does this work? What is happening here? + // If requesting specific fields, items and query association to said + // records are stored by ID reference. Thus, fields must always include + // the ID. + query = { + ...query, + _fields: [ + ...new Set( [ + ...( getNormalizedCommaSeparable( query._fields ) || + [] ), + entityConfig.key || DEFAULT_ENTITY_KEY, + ] ), + ].join(), + }; + } + + // Disable reason: While true that an early return could leave `path` + // unused, it's important that path is derived using the query prior to + // additional query modifications in the condition below, since those + // modifications are relevant to how the data is tracked in state, and not + // for how the request is made to the REST API. + + // eslint-disable-next-line @wordpress/no-unused-vars-before-return + const path = addQueryArgs( + entityConfig.baseURL + ( key ? '/' + key : '' ), + { + ...entityConfig.baseURLParams, + ...query, + } + ); + + if ( query !== undefined && query._fields ) { + query = { ...query, include: [ key ] }; + + // The resolution cache won't consider query as reusable based on the + // fields, so it's tested here, prior to initiating the REST request, + // and without causing `getEntityRecords` resolution to occur. + const hasRecords = select.hasEntityRecords( kind, name, query ); + if ( hasRecords ) { + return; + } + } + + const response = await apiFetch( { path, parse: false } ); + const record = await response.json(); + const permissions = getUserPermissionsFromAllowHeader( + response.headers?.get( 'allow' ) + ); + + const canUserResolutionsArgs = []; + const receiveUserPermissionArgs = {}; + for ( const action of ALLOWED_RESOURCE_ACTIONS ) { + receiveUserPermissionArgs[ + getUserPermissionCacheKey( action, { + kind, + name, + id: key, + } ) + ] = permissions[ action ]; + + canUserResolutionsArgs.push( [ + action, + { kind, name, id: key }, + ] ); + } + if ( window.__experimentalEnableSync && entityConfig.syncConfig && ! query ) { if ( globalThis.IS_GUTENBERG_PLUGIN ) { - const objectId = entityConfig.getSyncObjectId( key ); - // Loads the persisted document. await getSyncProvider().bootstrap( - entityConfig.syncObjectType, - objectId, - ( record ) => { - dispatch.receiveEntityRecords( - kind, - name, - record, - query - ); - } - ); - - // Bootstraps the edited document as well (and load from peers). - await getSyncProvider().bootstrap( - entityConfig.syncObjectType + '--edit', - objectId, - ( record ) => { + entityConfig.syncConfig, + record, + ( edits ) => { dispatch( { type: 'EDIT_ENTITY_RECORD', kind, name, recordId: key, - edits: record, + edits, meta: { undo: undefined, }, @@ -123,89 +171,13 @@ export const getEntityRecord = } ); } - } else { - if ( query !== undefined && query._fields ) { - // If requesting specific fields, items and query association to said - // records are stored by ID reference. Thus, fields must always include - // the ID. - query = { - ...query, - _fields: [ - ...new Set( [ - ...( getNormalizedCommaSeparable( - query._fields - ) || [] ), - entityConfig.key || DEFAULT_ENTITY_KEY, - ] ), - ].join(), - }; - } - - // Disable reason: While true that an early return could leave `path` - // unused, it's important that path is derived using the query prior to - // additional query modifications in the condition below, since those - // modifications are relevant to how the data is tracked in state, and not - // for how the request is made to the REST API. - - // eslint-disable-next-line @wordpress/no-unused-vars-before-return - const path = addQueryArgs( - entityConfig.baseURL + ( key ? '/' + key : '' ), - { - ...entityConfig.baseURLParams, - ...query, - } - ); - - if ( query !== undefined && query._fields ) { - query = { ...query, include: [ key ] }; - - // The resolution cache won't consider query as reusable based on the - // fields, so it's tested here, prior to initiating the REST request, - // and without causing `getEntityRecords` resolution to occur. - const hasRecords = select.hasEntityRecords( - kind, - name, - query - ); - if ( hasRecords ) { - return; - } - } - - const response = await apiFetch( { path, parse: false } ); - const record = await response.json(); - const permissions = getUserPermissionsFromAllowHeader( - response.headers?.get( 'allow' ) - ); - - const canUserResolutionsArgs = []; - const receiveUserPermissionArgs = {}; - for ( const action of ALLOWED_RESOURCE_ACTIONS ) { - receiveUserPermissionArgs[ - getUserPermissionCacheKey( action, { - kind, - name, - id: key, - } ) - ] = permissions[ action ]; - - canUserResolutionsArgs.push( [ - action, - { kind, name, id: key }, - ] ); - } - - registry.batch( () => { - dispatch.receiveEntityRecords( kind, name, record, query ); - dispatch.receiveUserPermissions( - receiveUserPermissionArgs - ); - dispatch.finishResolutions( - 'canUser', - canUserResolutionsArgs - ); - } ); } + + registry.batch( () => { + dispatch.receiveEntityRecords( kind, name, record, query ); + dispatch.receiveUserPermissions( receiveUserPermissionArgs ); + dispatch.finishResolutions( 'canUser', canUserResolutionsArgs ); + } ); } finally { dispatch.__unstableReleaseStoreLock( lock ); } diff --git a/packages/core-data/src/sync.ts b/packages/core-data/src/sync.ts index a9295508c1ecbc..0e54c45ef858fe 100644 --- a/packages/core-data/src/sync.ts +++ b/packages/core-data/src/sync.ts @@ -26,11 +26,11 @@ export function getSyncProvider(): SyncProvider { return syncProvider; } - const fallbackNoOpSyncProvider = { + const fallbackNoOpSyncProvider: SyncProvider = { __fallback: true, bootstrap: async () => {}, + configs: new Map(), discard: async () => {}, - register: () => {}, update: () => {}, }; diff --git a/packages/core-data/tsconfig.json b/packages/core-data/tsconfig.json index 31800502dae598..da362841c47955 100644 --- a/packages/core-data/tsconfig.json +++ b/packages/core-data/tsconfig.json @@ -3,7 +3,8 @@ "extends": "../../tsconfig.base.json", "compilerOptions": { "checkJs": false, - "noImplicitAny": false + "noImplicitAny": false, + "types": [ "node" ] }, "references": [ { "path": "../api-fetch" }, diff --git a/packages/dependency-extraction-webpack-plugin/lib/util.js b/packages/dependency-extraction-webpack-plugin/lib/util.js index b5c9f9057c2052..11d24cb433026a 100644 --- a/packages/dependency-extraction-webpack-plugin/lib/util.js +++ b/packages/dependency-extraction-webpack-plugin/lib/util.js @@ -8,7 +8,6 @@ const BUNDLED_PACKAGES = [ '@wordpress/dataviews/wp', '@wordpress/icons', '@wordpress/interface', - '@wordpress/sync', '@wordpress/undo-manager', '@wordpress/upload-media', '@wordpress/fields', diff --git a/packages/sync/CODE.md b/packages/sync/CODE.md index ae27b2541716dc..d61dac12824e89 100644 --- a/packages/sync/CODE.md +++ b/packages/sync/CODE.md @@ -1,6 +1,6 @@ # Status of the sync experiment in Gutenberg -The sync package is part of an ongoing research effort to lay the groundwork of Real-Time Collaboration in Gutenberg. +The sync package is part of an ongoing effort to lay the groundwork of Real-Time Collaboration in Gutenberg. Relevant docs: diff --git a/packages/sync/package.json b/packages/sync/package.json index 772979b2907c37..dea0587633994e 100644 --- a/packages/sync/package.json +++ b/packages/sync/package.json @@ -29,16 +29,15 @@ "types": "build-types", "sideEffects": false, "dependencies": { - "@babel/runtime": "7.25.7", "@types/simple-peer": "^9.11.5", + "@wordpress/hooks": "file:../hooks", "@wordpress/url": "file:../url", - "import-locals": "^2.0.0", - "lib0": "^0.2.42", + "lib0": "^0.2.99", "simple-peer": "^9.11.0", - "y-indexeddb": "~9.0.11", + "y-indexeddb": "^9.0.11", "y-protocols": "^1.0.5", - "y-webrtc": "~10.2.5", - "yjs": "~13.6.6" + "y-webrtc": "^10.2.5", + "yjs": "13.6.27" }, "publishConfig": { "access": "public" diff --git a/packages/sync/src/connect-indexdb.js b/packages/sync/src/connect-indexdb.js index 5523640408f575..c7ef7261fadf87 100644 --- a/packages/sync/src/connect-indexdb.js +++ b/packages/sync/src/connect-indexdb.js @@ -8,7 +8,7 @@ import { IndexeddbPersistence } from 'y-indexeddb'; /** @typedef {import('./types').ObjectID} ObjectID */ /** @typedef {import('./types').CRDTDoc} CRDTDoc */ /** @typedef {import('./types').ConnectDoc} ConnectDoc */ -/** @typedef {import('./types').SyncProvider} SyncProvider */ +/** @typedef {import('./types').ConnectDocResult} ConnectDocResult */ /** * Connect function to the IndexedDB persistence provider. @@ -17,15 +17,14 @@ import { IndexeddbPersistence } from 'y-indexeddb'; * @param {ObjectType} objectType The object type. * @param {CRDTDoc} doc The CRDT document. * - * @return {Promise<() => void>} Promise that resolves when the connection is established. + * @return {Promise< ConnectDocResult >} Promise that resolves when the connection is established. */ export function connectIndexDb( objectId, objectType, doc ) { const roomName = `${ objectType }-${ objectId }`; const provider = new IndexeddbPersistence( roomName, doc ); - return new Promise( ( resolve ) => { - provider.on( 'synced', () => { - resolve( () => provider.destroy() ); - } ); + return Promise.resolve( { + awareness: null, + destroy: () => provider.destroy(), } ); } diff --git a/packages/sync/src/create-webrtc-connection.ts b/packages/sync/src/create-webrtc-connection.ts index 456b68694b5da2..f32b9e31675bf5 100644 --- a/packages/sync/src/create-webrtc-connection.ts +++ b/packages/sync/src/create-webrtc-connection.ts @@ -35,6 +35,9 @@ export function createWebRTCConnection( { password, } ); - return Promise.resolve( () => true ); + return Promise.resolve( { + awareness: null, + destroy: () => {}, + } ); }; } diff --git a/packages/sync/src/index.ts b/packages/sync/src/index.ts index 8179d787ef86cc..cbf4ebeb3b8564 100644 --- a/packages/sync/src/index.ts +++ b/packages/sync/src/index.ts @@ -10,11 +10,25 @@ import { createWebRTCConnection } from './create-webrtc-connection'; import { createSyncProvider } from './provider'; import type { SyncProvider } from './types'; +export * as Y from 'yjs'; export { connectIndexDb } from './connect-indexdb'; export { createWebRTCConnection } from './create-webrtc-connection'; export { createSyncProvider } from './provider'; export * from './types'; +declare global { + interface Window { + __experimentalCollaborativeEditingSecret?: string; + wp: { + ajax: { + settings: { + url: string; + }; + }; + }; + } +} + /** * Returns a WebRTC sync provider. This is the curent default sync provider. * diff --git a/packages/sync/src/provider.js b/packages/sync/src/provider.js deleted file mode 100644 index 0be1dedab5d308..00000000000000 --- a/packages/sync/src/provider.js +++ /dev/null @@ -1,128 +0,0 @@ -/** - * External dependencies - */ -// @ts-ignore -import * as Y from 'yjs'; - -/** @typedef {import('./types').ObjectType} ObjectType */ -/** @typedef {import('./types').ObjectID} ObjectID */ -/** @typedef {import('./types').ObjectConfig} ObjectConfig */ -/** @typedef {import('./types').CRDTDoc} CRDTDoc */ -/** @typedef {import('./types').ConnectDoc} ConnectDoc */ -/** @typedef {import('./types').SyncProvider} SyncProvider */ - -/** - * Create a sync provider. - * - * @param {ConnectDoc} connectLocal Connect the document to a local database. - * @param {ConnectDoc} connectRemote Connect the document to a remote sync connection. - * @return {SyncProvider} Sync provider. - */ -export const createSyncProvider = ( connectLocal, connectRemote ) => { - /** - * @type {Record} - */ - const config = {}; - - /** - * @type {Recordvoid>>} - */ - const listeners = {}; - - /** - * @type {Record>} - */ - const docs = {}; - - /** - * Registers an object type. - * - * @param {ObjectType} objectType Object type to register. - * @param {ObjectConfig} objectConfig Object config. - */ - function register( objectType, objectConfig ) { - config[ objectType ] = objectConfig; - } - - /** - * Fetch data from local database or remote source. - * - * @param {ObjectType} objectType Object type to load. - * @param {ObjectID} objectId Object ID to load. - * @param {Function} handleChanges Callback to call when data changes. - */ - async function bootstrap( objectType, objectId, handleChanges ) { - const doc = new Y.Doc(); - docs[ objectType ] = docs[ objectType ] || {}; - docs[ objectType ][ objectId ] = doc; - - const updateHandler = () => { - const data = config[ objectType ].fromCRDTDoc( doc ); - handleChanges( data ); - }; - doc.on( 'update', updateHandler ); - - // connect to locally saved database. - const destroyLocalConnection = await connectLocal( - objectId, - objectType, - doc - ); - - // Once the database syncing is done, start the remote syncing - if ( connectRemote ) { - await connectRemote( objectId, objectType, doc ); - } - - const loadRemotely = config[ objectType ].fetch; - if ( loadRemotely ) { - loadRemotely( objectId ).then( ( data ) => { - doc.transact( () => { - config[ objectType ].applyChangesToDoc( doc, data ); - } ); - } ); - } - - listeners[ objectType ] = listeners[ objectType ] || {}; - listeners[ objectType ][ objectId ] = () => { - destroyLocalConnection(); - doc.off( 'update', updateHandler ); - }; - } - - /** - * Fetch data from local database or remote source. - * - * @param {ObjectType} objectType Object type to load. - * @param {ObjectID} objectId Object ID to load. - * @param {any} data Updates to make. - */ - async function update( objectType, objectId, data ) { - const doc = docs[ objectType ][ objectId ]; - if ( ! doc ) { - throw 'Error doc ' + objectType + ' ' + objectId + ' not found'; - } - doc.transact( () => { - config[ objectType ].applyChangesToDoc( doc, data ); - } ); - } - - /** - * Stop updating a document and discard it. - * - * @param {ObjectType} objectType Object type to load. - * @param {ObjectID} objectId Object ID to load. - */ - async function discard( objectType, objectId ) { - if ( listeners?.[ objectType ]?.[ objectId ] ) { - listeners[ objectType ][ objectId ](); - } - } - - return { - register, - bootstrap, - update, - discard, - }; -}; diff --git a/packages/sync/src/provider.ts b/packages/sync/src/provider.ts new file mode 100644 index 00000000000000..e80a99814d930d --- /dev/null +++ b/packages/sync/src/provider.ts @@ -0,0 +1,258 @@ +/** + * WordPress dependencies + */ + +/** + * External dependencies + */ +import type { Awareness } from 'y-protocols/awareness'; +import { removeAwarenessStates as removeAwarenessStatesFromProtocol } from 'y-protocols/awareness'; +import * as Y from 'yjs'; + +/** + * Internal dependencies + */ +import type { + AwarenessClientID, + AwarenessEventListener, + AwarenessStates, + ConnectDoc, + ConnectDocResult, + ObjectID, + ObjectData, + ObjectType, + SyncConfig, + SyncProvider, +} from './types'; + +interface EntityState { + awareness: Awareness | null; + destroy: () => void; + prevContentClientId: AwarenessClientID; + ydoc: Y.Doc; +} + +interface PendingAwarenessSetup { + pendingListeners: [ string, AwarenessEventListener ][]; + pendingStateFields: Map< string, unknown >; +} + +/** + * Create a sync provider. + * + * @param {ConnectDoc | null} connectLocal Connect the document to a local database. + * @param {ConnectDoc | null} connectRemote Connect the document to a remote sync connection. + * @return {SyncProvider} Sync provider. + */ +export const createSyncProvider = ( + connectLocal: ConnectDoc | null, + connectRemote: ConnectDoc | null +): SyncProvider => { + const configs: Map< ObjectType, SyncConfig > = new Map< + ObjectType, + SyncConfig + >(); + const entityStates: Map< string, EntityState > = new Map< + string, + EntityState + >(); + + const pendingAwarenessSetup: PendingAwarenessSetup = { + pendingListeners: [], + pendingStateFields: new Map< string, unknown >(), + }; + + /** + * Fetch data from local database or remote source. + * + * @param {SyncConfig} syncConfig Sync configuration for the object type. + * @param {ObjectData} initialData Initial data to apply to the document. + * @param {Function} handleChanges Callback to call when data changes. + */ + async function bootstrap( + syncConfig: SyncConfig, + initialData: ObjectData, + handleChanges: ( data: Partial< ObjectData > ) => void + ): Promise< void > { + const ydoc = new Y.Doc( { meta: new Map() } ); + const objectId = syncConfig.getObjectId( initialData ); + const objectType = syncConfig.objectType; + const entityId = `${ objectType }_${ objectId }`; + + configs.set( objectType, syncConfig ); + + const updateHandler: ( _update: Uint8Array, origin: string ) => void = ( + _update, + origin + ): void => { + if ( origin !== 'gutenberg' ) { + const data = syncConfig.fromCRDTDoc( ydoc ); + handleChanges( data ); + } + }; + + ydoc.on( 'update', updateHandler ); + + const connectLocalResult: ConnectDocResult | null = + ( await connectLocal?.( objectId, objectType, ydoc ) ) ?? null; + const connectRemoteResult = + ( await connectRemote?.( objectId, objectType, ydoc ) ) ?? null; + + const entityState: EntityState = { + awareness: connectRemoteResult?.awareness || null, + destroy: () => { + connectLocalResult?.destroy?.(); + connectRemoteResult?.destroy?.(); + + ydoc.off( 'update', updateHandler ); + ydoc.destroy(); + entityStates.delete( entityId ); + }, + prevContentClientId: 0, + ydoc, + }; + + entityStates.set( entityId, entityState ); + + bootstrapAwareness( connectRemoteResult?.awareness ?? null ); + update( objectType, initialData, initialData, 'gutenberg' ); + } + + /** + * Fetch data from local database or remote source. + * + * @param {ObjectType} objectType Object type to load. + * @param {ObjectData} record Record to load. + * @param {Partial< ObjectData >} changes Updates to make. + * @param {string} origin The source of change. + */ + function update( + objectType: ObjectType, + record: ObjectData, + changes: Partial< ObjectData >, + origin: string + ) { + const objectId = configs.get( objectType )?.getObjectId( record ); + const entityId = `${ objectType }_${ objectId }`; + const entityState = entityStates.get( entityId ); + + if ( ! entityState ) { + throw new Error( + `Entity ${ objectType }:${ objectId } not found ` + ); + } + + entityState.ydoc.transact( () => { + configs + .get( objectType ) + ?.applyChangesToDoc( entityState.ydoc, changes ); + }, origin ); + } + + /** + * Stop updating a document and discard it. + * + * @param {ObjectType} objectType Object type to load. + * @param {ObjectID} objectId Object ID to load. + */ + function discard( objectType: ObjectType, objectId: ObjectID ) { + const entityId = `${ objectType }_${ objectId }`; + + entityStates.get( entityId )?.destroy(); + entityStates.delete( entityId ); + } + + // Awareness handlers + + /** + * Add a listener for awareness events. + * + * @param {'update'|'change'} eventType Event type. + * @param {AwarenessEventListener} awarenessEventListener Awareness event listener. + */ + function addListener( + eventType: 'update' | 'change', + awarenessEventListener: AwarenessEventListener + ) { + Array.from( entityStates.values() ).forEach( ( entityState ) => { + entityState.awareness?.on( eventType, awarenessEventListener ); + } ); + + pendingAwarenessSetup.pendingListeners.push( [ + eventType, + awarenessEventListener, + ] ); + } + + function bootstrapAwareness( awareness: Awareness | null ) { + if ( ! awareness ) { + return; + } + + pendingAwarenessSetup.pendingListeners.forEach( + ( [ eventType, listener ]: [ string, AwarenessEventListener ] ) => { + awareness.on( eventType, listener ); + } + ); + + Array.from( + pendingAwarenessSetup.pendingStateFields.entries() + ).forEach( ( [ field, value ]: [ string, unknown ] ) => { + awareness.setLocalStateField( field, value ); + } ); + } + + /** + * Get the states of all awareness documents. + */ + function getStates(): AwarenessStates { + return ( + Array.from( entityStates.values() ) + .find( ( entityState ) => entityState.awareness ) + ?.awareness?.getStates() ?? new Map() + ); + } + + /** + * Removes the states of all awareness documents. + */ + function removeStates(): void { + Array.from( entityStates.values() ).forEach( ( entityState ) => { + if ( entityState.awareness ) { + removeAwarenessStatesFromProtocol( + entityState.awareness, + [ entityState.awareness.clientID ], + 'removeAwarenessStates' + ); + } + } ); + } + + /** + * Set a local state field on all awareness documents. + * + * @param {string} field Field name. + * @param {any} value State value. + */ + function setLocalState( field: string, value: unknown ) { + Array.from( entityStates.values() ).forEach( ( entityState ) => { + entityState.awareness?.setLocalStateField( field, value ); + } ); + + pendingAwarenessSetup.pendingStateFields.set( field, value ); + } + + return { + bootstrap, + configs, + discard, + update, + + awarenessManager: { + addListener, + getStates, + removeStates, + setLocalState, + }, + }; +}; diff --git a/packages/sync/src/types.ts b/packages/sync/src/types.ts index c141b3e17d6eb8..9526d0002ab587 100644 --- a/packages/sync/src/types.ts +++ b/packages/sync/src/types.ts @@ -1,44 +1,71 @@ /** * External dependencies */ +import type { Awareness } from 'y-protocols/awareness'; import type * as Y from 'yjs'; -declare global { - interface Window { - __experimentalCollaborativeEditingSecret?: string; - wp: { - ajax: { - settings: { - url: string; - }; - }; - }; - } -} - +export type * as Y from 'yjs'; export type ObjectID = string; export type ObjectType = string; +export type ObjectData = object; +export type UndoManager = Y.UndoManager; + +export type AwarenessClientID = number; + +export type AwarenessEventListener = ( params: { + added: AwarenessClientID[]; + updated: AwarenessClientID[]; + removed: AwarenessClientID[]; +} ) => void; + +export type AwarenessStates = Map< + AwarenessClientID, + Record< string, unknown > +>; + export type CRDTDoc = Y.Doc; -export type ObjectConfig = { - applyChangesToDoc: ( doc: CRDTDoc, data: object ) => void; - fetch: ( id: ObjectID ) => Promise< object >; - fromCRDTDoc: ( doc: CRDTDoc ) => any; +export type ConnectDocResult = { + awareness: Awareness | null; + destroy: () => void; }; export type ConnectDoc = ( id: ObjectID, type: ObjectType, - doc: CRDTDoc -) => Promise< () => void >; + ydoc: Y.Doc +) => Promise< ConnectDocResult >; + +export type SyncConfig = { + applyChangesToDoc: ( ydoc: Y.Doc, data: Partial< ObjectData > ) => void; + fromCRDTDoc: ( ydoc: Y.Doc ) => ObjectData; + getObjectId: ( data: ObjectData ) => ObjectID; + objectType: ObjectType; +}; export type SyncProvider = { + __fallback?: boolean; bootstrap: ( - type: ObjectType, - id: ObjectID, - handleChanges: ( data: object ) => void + syncConfig: SyncConfig, + initialData: ObjectData, + handleChanges: ( data: Partial< ObjectData > ) => void ) => Promise< void >; - discard: ( type: ObjectType, id: ObjectID ) => Promise< void >; - register: ( type: ObjectType, config: ObjectConfig ) => void; - update: ( type: ObjectType, id: ObjectID, data: object ) => void; + configs: Map< ObjectType, SyncConfig >; + discard: ( type: ObjectType, id: ObjectID ) => void; + update: ( + type: ObjectType, + record: ObjectData, + changes: Partial< ObjectData >, + origin: string + ) => void; + + awarenessManager?: { + addListener: ( + eventType: 'update' | 'change', + listener: AwarenessEventListener + ) => void; + getStates: () => AwarenessStates; + setLocalState: ( field: string, value: unknown ) => void; + removeStates: () => void; + }; }; diff --git a/tools/webpack/packages.js b/tools/webpack/packages.js index c99c25ee0127ce..f847d67e01f455 100644 --- a/tools/webpack/packages.js +++ b/tools/webpack/packages.js @@ -39,7 +39,6 @@ const BUNDLED_PACKAGES = [ '@wordpress/dataviews/wp', '@wordpress/icons', '@wordpress/interface', - '@wordpress/sync', '@wordpress/undo-manager', '@wordpress/upload-media', '@wordpress/fields', From b0d09f7fb88e3041105dfe82568bd24ffb48f7b6 Mon Sep 17 00:00:00 2001 From: chriszarate Date: Tue, 29 Jul 2025 16:39:25 -0400 Subject: [PATCH 12/54] Remove awareness implementation --- packages/sync/src/provider.ts | 107 ---------------------------------- packages/sync/src/types.ts | 25 -------- 2 files changed, 132 deletions(-) diff --git a/packages/sync/src/provider.ts b/packages/sync/src/provider.ts index e80a99814d930d..d590354d898d60 100644 --- a/packages/sync/src/provider.ts +++ b/packages/sync/src/provider.ts @@ -5,17 +5,12 @@ /** * External dependencies */ -import type { Awareness } from 'y-protocols/awareness'; -import { removeAwarenessStates as removeAwarenessStatesFromProtocol } from 'y-protocols/awareness'; import * as Y from 'yjs'; /** * Internal dependencies */ import type { - AwarenessClientID, - AwarenessEventListener, - AwarenessStates, ConnectDoc, ConnectDocResult, ObjectID, @@ -26,17 +21,10 @@ import type { } from './types'; interface EntityState { - awareness: Awareness | null; destroy: () => void; - prevContentClientId: AwarenessClientID; ydoc: Y.Doc; } -interface PendingAwarenessSetup { - pendingListeners: [ string, AwarenessEventListener ][]; - pendingStateFields: Map< string, unknown >; -} - /** * Create a sync provider. * @@ -57,11 +45,6 @@ export const createSyncProvider = ( EntityState >(); - const pendingAwarenessSetup: PendingAwarenessSetup = { - pendingListeners: [], - pendingStateFields: new Map< string, unknown >(), - }; - /** * Fetch data from local database or remote source. * @@ -99,7 +82,6 @@ export const createSyncProvider = ( ( await connectRemote?.( objectId, objectType, ydoc ) ) ?? null; const entityState: EntityState = { - awareness: connectRemoteResult?.awareness || null, destroy: () => { connectLocalResult?.destroy?.(); connectRemoteResult?.destroy?.(); @@ -108,13 +90,11 @@ export const createSyncProvider = ( ydoc.destroy(); entityStates.delete( entityId ); }, - prevContentClientId: 0, ydoc, }; entityStates.set( entityId, entityState ); - bootstrapAwareness( connectRemoteResult?.awareness ?? null ); update( objectType, initialData, initialData, 'gutenberg' ); } @@ -162,97 +142,10 @@ export const createSyncProvider = ( entityStates.delete( entityId ); } - // Awareness handlers - - /** - * Add a listener for awareness events. - * - * @param {'update'|'change'} eventType Event type. - * @param {AwarenessEventListener} awarenessEventListener Awareness event listener. - */ - function addListener( - eventType: 'update' | 'change', - awarenessEventListener: AwarenessEventListener - ) { - Array.from( entityStates.values() ).forEach( ( entityState ) => { - entityState.awareness?.on( eventType, awarenessEventListener ); - } ); - - pendingAwarenessSetup.pendingListeners.push( [ - eventType, - awarenessEventListener, - ] ); - } - - function bootstrapAwareness( awareness: Awareness | null ) { - if ( ! awareness ) { - return; - } - - pendingAwarenessSetup.pendingListeners.forEach( - ( [ eventType, listener ]: [ string, AwarenessEventListener ] ) => { - awareness.on( eventType, listener ); - } - ); - - Array.from( - pendingAwarenessSetup.pendingStateFields.entries() - ).forEach( ( [ field, value ]: [ string, unknown ] ) => { - awareness.setLocalStateField( field, value ); - } ); - } - - /** - * Get the states of all awareness documents. - */ - function getStates(): AwarenessStates { - return ( - Array.from( entityStates.values() ) - .find( ( entityState ) => entityState.awareness ) - ?.awareness?.getStates() ?? new Map() - ); - } - - /** - * Removes the states of all awareness documents. - */ - function removeStates(): void { - Array.from( entityStates.values() ).forEach( ( entityState ) => { - if ( entityState.awareness ) { - removeAwarenessStatesFromProtocol( - entityState.awareness, - [ entityState.awareness.clientID ], - 'removeAwarenessStates' - ); - } - } ); - } - - /** - * Set a local state field on all awareness documents. - * - * @param {string} field Field name. - * @param {any} value State value. - */ - function setLocalState( field: string, value: unknown ) { - Array.from( entityStates.values() ).forEach( ( entityState ) => { - entityState.awareness?.setLocalStateField( field, value ); - } ); - - pendingAwarenessSetup.pendingStateFields.set( field, value ); - } - return { bootstrap, configs, discard, update, - - awarenessManager: { - addListener, - getStates, - removeStates, - setLocalState, - }, }; }; diff --git a/packages/sync/src/types.ts b/packages/sync/src/types.ts index 9526d0002ab587..e33ae936e41037 100644 --- a/packages/sync/src/types.ts +++ b/packages/sync/src/types.ts @@ -1,7 +1,6 @@ /** * External dependencies */ -import type { Awareness } from 'y-protocols/awareness'; import type * as Y from 'yjs'; export type * as Y from 'yjs'; @@ -10,23 +9,9 @@ export type ObjectType = string; export type ObjectData = object; export type UndoManager = Y.UndoManager; -export type AwarenessClientID = number; - -export type AwarenessEventListener = ( params: { - added: AwarenessClientID[]; - updated: AwarenessClientID[]; - removed: AwarenessClientID[]; -} ) => void; - -export type AwarenessStates = Map< - AwarenessClientID, - Record< string, unknown > ->; - export type CRDTDoc = Y.Doc; export type ConnectDocResult = { - awareness: Awareness | null; destroy: () => void; }; @@ -58,14 +43,4 @@ export type SyncProvider = { changes: Partial< ObjectData >, origin: string ) => void; - - awarenessManager?: { - addListener: ( - eventType: 'update' | 'change', - listener: AwarenessEventListener - ) => void; - getStates: () => AwarenessStates; - setLocalState: ( field: string, value: unknown ) => void; - removeStates: () => void; - }; }; From 4e01172f5dc63eade8d1e5eddb5cabb36ff9b17d Mon Sep 17 00:00:00 2001 From: chriszarate Date: Wed, 30 Jul 2025 13:46:33 -0400 Subject: [PATCH 13/54] Convert SyncProvider to class --- packages/core-data/src/sync.ts | 11 +-- packages/sync/src/index.ts | 7 +- packages/sync/src/provider.ts | 127 ++++++++++++++++++++------------- packages/sync/src/types.ts | 17 ----- 4 files changed, 81 insertions(+), 81 deletions(-) diff --git a/packages/core-data/src/sync.ts b/packages/core-data/src/sync.ts index 0e54c45ef858fe..b52faff570dc4c 100644 --- a/packages/core-data/src/sync.ts +++ b/packages/core-data/src/sync.ts @@ -2,8 +2,7 @@ * WordPress dependencies */ import { applyFilters } from '@wordpress/hooks'; -import { getWebRTCSyncProvider } from '@wordpress/sync'; -import type { SyncProvider } from '@wordpress/sync'; +import { getWebRTCSyncProvider, SyncProvider } from '@wordpress/sync'; declare global { interface Window { @@ -26,13 +25,7 @@ export function getSyncProvider(): SyncProvider { return syncProvider; } - const fallbackNoOpSyncProvider: SyncProvider = { - __fallback: true, - bootstrap: async () => {}, - configs: new Map(), - discard: async () => {}, - update: () => {}, - }; + const fallbackNoOpSyncProvider = new SyncProvider( null, null ); syncProvider = applyFilters( 'core.getSyncProvider', diff --git a/packages/sync/src/index.ts b/packages/sync/src/index.ts index cbf4ebeb3b8564..a630969d6d49b8 100644 --- a/packages/sync/src/index.ts +++ b/packages/sync/src/index.ts @@ -7,13 +7,12 @@ */ import { connectIndexDb } from './connect-indexdb'; import { createWebRTCConnection } from './create-webrtc-connection'; -import { createSyncProvider } from './provider'; -import type { SyncProvider } from './types'; +import { SyncProvider } from './provider'; export * as Y from 'yjs'; export { connectIndexDb } from './connect-indexdb'; export { createWebRTCConnection } from './create-webrtc-connection'; -export { createSyncProvider } from './provider'; +export { SyncProvider } from './provider'; export * from './types'; declare global { @@ -35,7 +34,7 @@ declare global { * @return {SyncProvider} The WebRTC sync provider. */ export function getWebRTCSyncProvider(): SyncProvider { - return createSyncProvider( + return new SyncProvider( connectIndexDb, createWebRTCConnection( { password: window?.__experimentalCollaborativeEditingSecret, diff --git a/packages/sync/src/provider.ts b/packages/sync/src/provider.ts index d590354d898d60..010ecfd773e145 100644 --- a/packages/sync/src/provider.ts +++ b/packages/sync/src/provider.ts @@ -1,7 +1,3 @@ -/** - * WordPress dependencies - */ - /** * External dependencies */ @@ -13,38 +9,46 @@ import * as Y from 'yjs'; import type { ConnectDoc, ConnectDocResult, + CRDTDoc, ObjectID, ObjectData, ObjectType, SyncConfig, - SyncProvider, } from './types'; interface EntityState { destroy: () => void; - ydoc: Y.Doc; + ydoc: CRDTDoc; } -/** - * Create a sync provider. - * - * @param {ConnectDoc | null} connectLocal Connect the document to a local database. - * @param {ConnectDoc | null} connectRemote Connect the document to a remote sync connection. - * @return {SyncProvider} Sync provider. - */ -export const createSyncProvider = ( - connectLocal: ConnectDoc | null, - connectRemote: ConnectDoc | null -): SyncProvider => { - const configs: Map< ObjectType, SyncConfig > = new Map< +export class SyncProvider { + protected connectLocal: ConnectDoc | null; + protected connectRemote: ConnectDoc | null; + + protected configs: Map< ObjectType, SyncConfig > = new Map< ObjectType, SyncConfig >(); - const entityStates: Map< string, EntityState > = new Map< + + protected entityStates: Map< string, EntityState > = new Map< string, EntityState >(); + /** + * Constructor. + * + * @param {ConnectDoc | null} connectLocal Connect the document to a local database. + * @param {ConnectDoc | null} connectRemote Connect the document to a remote sync connection. + */ + public constructor( + connectLocal: ConnectDoc | null, + connectRemote: ConnectDoc | null + ) { + this.connectLocal = connectLocal; + this.connectRemote = connectRemote; + } + /** * Fetch data from local database or remote source. * @@ -52,7 +56,7 @@ export const createSyncProvider = ( * @param {ObjectData} initialData Initial data to apply to the document. * @param {Function} handleChanges Callback to call when data changes. */ - async function bootstrap( + public async bootstrap( syncConfig: SyncConfig, initialData: ObjectData, handleChanges: ( data: Partial< ObjectData > ) => void @@ -60,9 +64,9 @@ export const createSyncProvider = ( const ydoc = new Y.Doc( { meta: new Map() } ); const objectId = syncConfig.getObjectId( initialData ); const objectType = syncConfig.objectType; - const entityId = `${ objectType }_${ objectId }`; + const entityId = this.getEntityId( objectType, objectId ); - configs.set( objectType, syncConfig ); + this.configs.set( objectType, syncConfig ); const updateHandler: ( _update: Uint8Array, origin: string ) => void = ( _update, @@ -77,9 +81,10 @@ export const createSyncProvider = ( ydoc.on( 'update', updateHandler ); const connectLocalResult: ConnectDocResult | null = - ( await connectLocal?.( objectId, objectType, ydoc ) ) ?? null; + ( await this.connectLocal?.( objectId, objectType, ydoc ) ) ?? null; const connectRemoteResult = - ( await connectRemote?.( objectId, objectType, ydoc ) ) ?? null; + ( await this.connectRemote?.( objectId, objectType, ydoc ) ) ?? + null; const entityState: EntityState = { destroy: () => { @@ -88,14 +93,43 @@ export const createSyncProvider = ( ydoc.off( 'update', updateHandler ); ydoc.destroy(); - entityStates.delete( entityId ); + this.entityStates.delete( entityId ); }, ydoc, }; - entityStates.set( entityId, entityState ); + this.entityStates.set( entityId, entityState ); - update( objectType, initialData, initialData, 'gutenberg' ); + this.update( objectType, initialData, initialData, 'gutenberg' ); + } + + /** + * Get the entity ID for the given object type and object ID. + * + * @param {ObjectType} objectType Object type. + * @param {ObjectID} objectId Object ID. + */ + protected getEntityId( + objectType: ObjectType, + objectId: ObjectID + ): string { + return `${ objectType }_${ objectId }`; + } + + /** + * Get the entity state for the given object type and object ID. + * + * @param {ObjectType} objectType Object type. + * @param {ObjectID} objectId Object ID. + */ + protected getEntityState( + objectType: ObjectType, + objectId: ObjectID + ): EntityState | null { + return ( + this.entityStates.get( this.getEntityId( objectType, objectId ) ) ?? + null + ); } /** @@ -106,24 +140,22 @@ export const createSyncProvider = ( * @param {Partial< ObjectData >} changes Updates to make. * @param {string} origin The source of change. */ - function update( + public update( objectType: ObjectType, record: ObjectData, changes: Partial< ObjectData >, origin: string - ) { - const objectId = configs.get( objectType )?.getObjectId( record ); - const entityId = `${ objectType }_${ objectId }`; - const entityState = entityStates.get( entityId ); + ): void { + const objectId = this.configs.get( objectType )?.getObjectId( record ); - if ( ! entityState ) { - throw new Error( - `Entity ${ objectType }:${ objectId } not found ` - ); + if ( ! objectId ) { + return; } - entityState.ydoc.transact( () => { - configs + const entityState = this.getEntityState( objectType, objectId ); + + entityState?.ydoc.transact( () => { + this.configs .get( objectType ) ?.applyChangesToDoc( entityState.ydoc, changes ); }, origin ); @@ -132,20 +164,13 @@ export const createSyncProvider = ( /** * Stop updating a document and discard it. * - * @param {ObjectType} objectType Object type to load. - * @param {ObjectID} objectId Object ID to load. + * @param {ObjectType} objectType Object type to discard. + * @param {ObjectID} objectId Object ID to discard. */ - function discard( objectType: ObjectType, objectId: ObjectID ) { + public discard( objectType: ObjectType, objectId: ObjectID ): void { const entityId = `${ objectType }_${ objectId }`; - entityStates.get( entityId )?.destroy(); - entityStates.delete( entityId ); + this.getEntityState( objectType, objectId )?.destroy(); + this.entityStates.delete( entityId ); } - - return { - bootstrap, - configs, - discard, - update, - }; -}; +} diff --git a/packages/sync/src/types.ts b/packages/sync/src/types.ts index e33ae936e41037..f4a502381a04ca 100644 --- a/packages/sync/src/types.ts +++ b/packages/sync/src/types.ts @@ -27,20 +27,3 @@ export type SyncConfig = { getObjectId: ( data: ObjectData ) => ObjectID; objectType: ObjectType; }; - -export type SyncProvider = { - __fallback?: boolean; - bootstrap: ( - syncConfig: SyncConfig, - initialData: ObjectData, - handleChanges: ( data: Partial< ObjectData > ) => void - ) => Promise< void >; - configs: Map< ObjectType, SyncConfig >; - discard: ( type: ObjectType, id: ObjectID ) => void; - update: ( - type: ObjectType, - record: ObjectData, - changes: Partial< ObjectData >, - origin: string - ) => void; -}; From b3cc7e3081914677154a0c22ab3f0f20441f617e Mon Sep 17 00:00:00 2001 From: ingeniumed Date: Fri, 1 Aug 2025 13:55:28 +1000 Subject: [PATCH 14/54] Update the package-lock.json and bump the y- packages --- package-lock.json | 181 ++++++++++++++++++++++--------------- packages/sync/package.json | 6 +- 2 files changed, 113 insertions(+), 74 deletions(-) diff --git a/package-lock.json b/package-lock.json index ba91ec4675d642..2dade0d883da63 100644 --- a/package-lock.json +++ b/package-lock.json @@ -48630,56 +48630,6 @@ "node": ">=0.4" } }, - "node_modules/y-indexeddb": { - "version": "9.0.11", - "resolved": "https://registry.npmjs.org/y-indexeddb/-/y-indexeddb-9.0.11.tgz", - "integrity": "sha512-HOKQ70qW1h2WJGtOKu9rE8fbX86ExVZedecndMuhwax3yM4DQsQzCTGHt/jvTrFZr/9Ahvd8neD6aZ4dMMjtdg==", - "dependencies": { - "lib0": "^0.2.74" - }, - "funding": { - "type": "GitHub Sponsors ❤", - "url": "https://github.com/sponsors/dmonad" - }, - "peerDependencies": { - "yjs": "^13.0.0" - } - }, - "node_modules/y-protocols": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/y-protocols/-/y-protocols-1.0.5.tgz", - "integrity": "sha512-Wil92b7cGk712lRHDqS4T90IczF6RkcvCwAD0A2OPg+adKmOe+nOiT/N2hvpQIWS3zfjmtL4CPaH5sIW1Hkm/A==", - "dependencies": { - "lib0": "^0.2.42" - }, - "funding": { - "type": "GitHub Sponsors ❤", - "url": "https://github.com/sponsors/dmonad" - } - }, - "node_modules/y-webrtc": { - "version": "10.2.5", - "resolved": "https://registry.npmjs.org/y-webrtc/-/y-webrtc-10.2.5.tgz", - "integrity": "sha512-ZyBNvTI5L28sQ2PQI0T/JvyWgvuTq05L21vGkIlcvNLNSJqAaLCBJRe3FHEqXoaogqWmRcEAKGfII4ErNXMnNw==", - "dependencies": { - "lib0": "^0.2.42", - "simple-peer": "^9.11.0", - "y-protocols": "^1.0.5" - }, - "bin": { - "y-webrtc-signaling": "bin/server.js" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "type": "GitHub Sponsors ❤", - "url": "https://github.com/sponsors/dmonad" - }, - "optionalDependencies": { - "ws": "^7.2.0" - } - }, "node_modules/y18n": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.3.tgz", @@ -48812,23 +48762,6 @@ "fd-slicer": "~1.1.0" } }, - "node_modules/yjs": { - "version": "13.6.6", - "resolved": "https://registry.npmjs.org/yjs/-/yjs-13.6.6.tgz", - "integrity": "sha512-VTvezMeMuOra9jKG1Ym5XuQ2H4xXOubIIIupv/B5oygasa9IqDE7Ufv93QTSe9uz69J5VZGMQb2WTEmJv4kJFQ==", - "license": "MIT", - "dependencies": { - "lib0": "^0.2.74" - }, - "engines": { - "node": ">=16.0.0", - "npm": ">=8.0.0" - }, - "funding": { - "type": "GitHub Sponsors ❤", - "url": "https://github.com/sponsors/dmonad" - } - }, "node_modules/yn": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", @@ -52310,16 +52243,122 @@ "@wordpress/url": "file:../url", "lib0": "^0.2.99", "simple-peer": "^9.11.0", - "y-indexeddb": "^9.0.11", - "y-protocols": "^1.0.5", - "y-webrtc": "^10.2.5", - "yjs": "13.6.6" + "y-indexeddb": "^9.0.12", + "y-protocols": "^1.0.6", + "y-webrtc": "^10.3.0", + "yjs": "13.6.27" }, "engines": { "node": ">=18.12.0", "npm": ">=8.19.2" } }, + "packages/sync/node_modules/ws": { + "version": "8.18.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.3.tgz", + "integrity": "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "packages/sync/node_modules/y-indexeddb": { + "version": "9.0.12", + "resolved": "https://registry.npmjs.org/y-indexeddb/-/y-indexeddb-9.0.12.tgz", + "integrity": "sha512-9oCFRSPPzBK7/w5vOkJBaVCQZKHXB/v6SIT+WYhnJxlEC61juqG0hBrAf+y3gmSMLFLwICNH9nQ53uscuse6Hg==", + "license": "MIT", + "dependencies": { + "lib0": "^0.2.74" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=8.0.0" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + }, + "peerDependencies": { + "yjs": "^13.0.0" + } + }, + "packages/sync/node_modules/y-protocols": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/y-protocols/-/y-protocols-1.0.6.tgz", + "integrity": "sha512-vHRF2L6iT3rwj1jub/K5tYcTT/mEYDUppgNPXwp8fmLpui9f7Yeq3OEtTLVF012j39QnV+KEQpNqoN7CWU7Y9Q==", + "license": "MIT", + "dependencies": { + "lib0": "^0.2.85" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=8.0.0" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + }, + "peerDependencies": { + "yjs": "^13.0.0" + } + }, + "packages/sync/node_modules/y-webrtc": { + "version": "10.3.0", + "resolved": "https://registry.npmjs.org/y-webrtc/-/y-webrtc-10.3.0.tgz", + "integrity": "sha512-KalJr7dCgUgyVFxoG3CQYbpS0O2qybegD0vI4bYnYHI0MOwoVbucED3RZ5f2o1a5HZb1qEssUKS0H/Upc6p1lA==", + "license": "MIT", + "dependencies": { + "lib0": "^0.2.42", + "simple-peer": "^9.11.0", + "y-protocols": "^1.0.6" + }, + "bin": { + "y-webrtc-signaling": "bin/server.js" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + }, + "optionalDependencies": { + "ws": "^8.14.2" + }, + "peerDependencies": { + "yjs": "^13.6.8" + } + }, + "packages/sync/node_modules/yjs": { + "version": "13.6.27", + "resolved": "https://registry.npmjs.org/yjs/-/yjs-13.6.27.tgz", + "integrity": "sha512-OIDwaflOaq4wC6YlPBy2L6ceKeKuF7DeTxx+jPzv1FHn9tCZ0ZwSRnUBxD05E3yed46fv/FWJbvR+Ud7x0L7zw==", + "license": "MIT", + "dependencies": { + "lib0": "^0.2.99" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=8.0.0" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + } + }, "packages/token-list": { "name": "@wordpress/token-list", "version": "3.27.0", diff --git a/packages/sync/package.json b/packages/sync/package.json index dea0587633994e..a5d90830acc44d 100644 --- a/packages/sync/package.json +++ b/packages/sync/package.json @@ -34,9 +34,9 @@ "@wordpress/url": "file:../url", "lib0": "^0.2.99", "simple-peer": "^9.11.0", - "y-indexeddb": "^9.0.11", - "y-protocols": "^1.0.5", - "y-webrtc": "^10.2.5", + "y-indexeddb": "^9.0.12", + "y-protocols": "^1.0.6", + "y-webrtc": "^10.3.0", "yjs": "13.6.27" }, "publishConfig": { From 76b62885020e668851dad806c01c4ec66beca93b Mon Sep 17 00:00:00 2001 From: chriszarate Date: Tue, 5 Aug 2025 14:14:34 -0400 Subject: [PATCH 15/54] Add awareness as optional property on ConnectDocResult --- packages/sync/src/connect-indexdb.js | 1 - packages/sync/src/create-webrtc-connection.ts | 1 - packages/sync/src/types.ts | 2 ++ 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/sync/src/connect-indexdb.js b/packages/sync/src/connect-indexdb.js index c7ef7261fadf87..5329f066b296ea 100644 --- a/packages/sync/src/connect-indexdb.js +++ b/packages/sync/src/connect-indexdb.js @@ -24,7 +24,6 @@ export function connectIndexDb( objectId, objectType, doc ) { const provider = new IndexeddbPersistence( roomName, doc ); return Promise.resolve( { - awareness: null, destroy: () => provider.destroy(), } ); } diff --git a/packages/sync/src/create-webrtc-connection.ts b/packages/sync/src/create-webrtc-connection.ts index f32b9e31675bf5..2ea51777e8a91e 100644 --- a/packages/sync/src/create-webrtc-connection.ts +++ b/packages/sync/src/create-webrtc-connection.ts @@ -36,7 +36,6 @@ export function createWebRTCConnection( { } ); return Promise.resolve( { - awareness: null, destroy: () => {}, } ); }; diff --git a/packages/sync/src/types.ts b/packages/sync/src/types.ts index f4a502381a04ca..ffa63c527a15fe 100644 --- a/packages/sync/src/types.ts +++ b/packages/sync/src/types.ts @@ -2,6 +2,7 @@ * External dependencies */ import type * as Y from 'yjs'; +import type { Awareness } from 'y-protocols/awareness'; export type * as Y from 'yjs'; export type ObjectID = string; @@ -12,6 +13,7 @@ export type UndoManager = Y.UndoManager; export type CRDTDoc = Y.Doc; export type ConnectDocResult = { + awareness?: Awareness; destroy: () => void; }; From 323eaabad23effec680de3fe6bb9f5f8aaeeea58 Mon Sep 17 00:00:00 2001 From: chriszarate Date: Tue, 5 Aug 2025 14:15:15 -0400 Subject: [PATCH 16/54] Update SyncProvider for better extensibility --- packages/sync/src/provider.ts | 80 ++++++++++++++++++----------------- packages/sync/src/types.ts | 1 + 2 files changed, 42 insertions(+), 39 deletions(-) diff --git a/packages/sync/src/provider.ts b/packages/sync/src/provider.ts index 010ecfd773e145..f87e47c1f41149 100644 --- a/packages/sync/src/provider.ts +++ b/packages/sync/src/provider.ts @@ -10,6 +10,7 @@ import type { ConnectDoc, ConnectDocResult, CRDTDoc, + EntityID, ObjectID, ObjectData, ObjectType, @@ -22,18 +23,12 @@ interface EntityState { } export class SyncProvider { - protected connectLocal: ConnectDoc | null; - protected connectRemote: ConnectDoc | null; + private connectLocal: ConnectDoc | null; + private connectRemote: ConnectDoc | null; - protected configs: Map< ObjectType, SyncConfig > = new Map< - ObjectType, - SyncConfig - >(); - - protected entityStates: Map< string, EntityState > = new Map< - string, - EntityState - >(); + protected configs: Map< ObjectType, SyncConfig > = new Map(); + protected connections: Map< EntityID, ConnectDocResult[] > = new Map(); + protected entityStates: Map< EntityID, EntityState > = new Map(); /** * Constructor. @@ -49,6 +44,26 @@ export class SyncProvider { this.connectRemote = connectRemote; } + /** + * Connect to a document. + * + * @param {ObjectID} objectId Object ID to connect. + * @param {ObjectType} objectType Object type to connect. + * @param {CRDTDoc} ydoc Yjs document for the object. + */ + private async connect( + objectId: ObjectID, + objectType: ObjectType, + ydoc: CRDTDoc + ): Promise< ConnectDocResult[] > { + return ( + await Promise.all( [ + this.connectLocal?.( objectId, objectType, ydoc ), + this.connectRemote?.( objectId, objectType, ydoc ), + ] ) + ).filter( ( result ): result is ConnectDocResult => Boolean( result ) ); + } + /** * Fetch data from local database or remote source. * @@ -64,41 +79,31 @@ export class SyncProvider { const ydoc = new Y.Doc( { meta: new Map() } ); const objectId = syncConfig.getObjectId( initialData ); const objectType = syncConfig.objectType; + const connections = await this.connect( objectId, objectType, ydoc ); const entityId = this.getEntityId( objectType, objectId ); - this.configs.set( objectType, syncConfig ); + const onDestroy = (): void => { + connections.forEach( ( result ) => result.destroy() ); + ydoc.off( 'update', onUpdate ); + ydoc.destroy(); + this.entityStates.delete( entityId ); + }; - const updateHandler: ( _update: Uint8Array, origin: string ) => void = ( - _update, - origin - ): void => { + const onUpdate = ( _update: Uint8Array, origin: string ): void => { if ( origin !== 'gutenberg' ) { const data = syncConfig.fromCRDTDoc( ydoc ); handleChanges( data ); } }; - ydoc.on( 'update', updateHandler ); - - const connectLocalResult: ConnectDocResult | null = - ( await this.connectLocal?.( objectId, objectType, ydoc ) ) ?? null; - const connectRemoteResult = - ( await this.connectRemote?.( objectId, objectType, ydoc ) ) ?? - null; - - const entityState: EntityState = { - destroy: () => { - connectLocalResult?.destroy?.(); - connectRemoteResult?.destroy?.(); + ydoc.on( 'update', onUpdate ); - ydoc.off( 'update', updateHandler ); - ydoc.destroy(); - this.entityStates.delete( entityId ); - }, + this.configs.set( objectType, syncConfig ); + this.connections.set( entityId, connections ); + this.entityStates.set( entityId, { ydoc, - }; - - this.entityStates.set( entityId, entityState ); + destroy: onDestroy, + } ); this.update( objectType, initialData, initialData, 'gutenberg' ); } @@ -112,7 +117,7 @@ export class SyncProvider { protected getEntityId( objectType: ObjectType, objectId: ObjectID - ): string { + ): EntityID { return `${ objectType }_${ objectId }`; } @@ -168,9 +173,6 @@ export class SyncProvider { * @param {ObjectID} objectId Object ID to discard. */ public discard( objectType: ObjectType, objectId: ObjectID ): void { - const entityId = `${ objectType }_${ objectId }`; - this.getEntityState( objectType, objectId )?.destroy(); - this.entityStates.delete( entityId ); } } diff --git a/packages/sync/src/types.ts b/packages/sync/src/types.ts index ffa63c527a15fe..bca2d1364112e7 100644 --- a/packages/sync/src/types.ts +++ b/packages/sync/src/types.ts @@ -5,6 +5,7 @@ import type * as Y from 'yjs'; import type { Awareness } from 'y-protocols/awareness'; export type * as Y from 'yjs'; +export type EntityID = string; export type ObjectID = string; export type ObjectType = string; export type ObjectData = object; From e9aabe231a0a8d829a51ff2f7efdc6f8fa1c8abe Mon Sep 17 00:00:00 2001 From: ingeniumed Date: Wed, 6 Aug 2025 14:12:59 +1000 Subject: [PATCH 17/54] Add the Yjs undo manager in the provider.ts --- .vscode/launch.json | 38 +++++++++++++++++++++++++++++++++++ packages/sync/src/provider.ts | 29 ++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000000000..768d17a8f09d98 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,38 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Listen for Xdebug", + "type": "php", + "request": "launch", + "port": 9003, + "pathMappings": { + "/var/www/html/wp-content/plugins/gutenberg": "${workspaceFolder}" + } + }, + { + "name": "Debug Vivaldi", + "type": "msedge", + "request": "launch", + "url": "http://localhost:8888/wp-admin", + "runtimeExecutable": "/Applications/Vivaldi.app/Contents/MacOS/Vivaldi", + }, + { + "type": "node", + "request": "launch", + "name": "Debug current e2e test", + "program": "${workspaceFolder}/node_modules/@wordpress/scripts/bin/wp-scripts.js", + "args": [ + "test-e2e", + "--config=${workspaceFolder}/packages/e2e-tests/jest.config.js", + "--verbose=true", + "--runInBand", + "--watch", + "${file}" + ], + "console": "integratedTerminal", + "internalConsoleOptions": "neverOpen", + "trace": "all" + } + ] +} diff --git a/packages/sync/src/provider.ts b/packages/sync/src/provider.ts index f87e47c1f41149..9d8a5139637ae7 100644 --- a/packages/sync/src/provider.ts +++ b/packages/sync/src/provider.ts @@ -18,6 +18,7 @@ import type { } from './types'; interface EntityState { + undoManager: Y.UndoManager; destroy: () => void; ydoc: CRDTDoc; } @@ -82,10 +83,22 @@ export class SyncProvider { const connections = await this.connect( objectId, objectType, ydoc ); const entityId = this.getEntityId( objectType, objectId ); + const undoManager = new Y.UndoManager( ydoc.getMap( 'document' ), { + // Ensure we undo and redo one character at a time. + captureTimeout: 0, + // Ensure that we only scope the undo/redo to the current client, and Gutenberg origins. + // ToDo: Keep an eye on this, as it needs to be battle tested. + trackedOrigins: new Set( [ 'gutenberg', ydoc.clientID ] ), + // This ensures that are able to improve the client specific undo/redo experience. + // This reduces the bugs we see, but it doesn't eliminate them entirely. + ignoreRemoteMapChanges: true, + } ); + const onDestroy = (): void => { connections.forEach( ( result ) => result.destroy() ); ydoc.off( 'update', onUpdate ); ydoc.destroy(); + undoManager.destroy(); this.entityStates.delete( entityId ); }; @@ -101,6 +114,7 @@ export class SyncProvider { this.configs.set( objectType, syncConfig ); this.connections.set( entityId, connections ); this.entityStates.set( entityId, { + undoManager, ydoc, destroy: onDestroy, } ); @@ -137,6 +151,21 @@ export class SyncProvider { ); } + /** + * Get the undo manager for the given object type and object ID. + * + * @param {ObjectType} objectType Object type. + * @param {ObjectID} objectId Object ID. + * @return {Y.UndoManager | null} The undo manager, or null if not found. + */ + public getUndoManager( + objectType: ObjectType, + objectId: ObjectID + ): Y.UndoManager | null { + const entityState = this.getEntityState( objectType, objectId ); + return entityState ? entityState.undoManager : null; + } + /** * Fetch data from local database or remote source. * From 8404bed3fabee0e292c780bd9b9900ead3045505 Mon Sep 17 00:00:00 2001 From: ingeniumed Date: Wed, 6 Aug 2025 14:56:09 +1000 Subject: [PATCH 18/54] Make a separate exported clas for the undo manager coming from the provider --- .vscode/launch.json | 2 +- packages/core-data/src/reducer.js | 3 + packages/sync/src/provider.ts | 10 +-- packages/sync/src/undo-manager.ts | 100 ++++++++++++++++++++++++++++++ 4 files changed, 110 insertions(+), 5 deletions(-) create mode 100644 packages/sync/src/undo-manager.ts diff --git a/.vscode/launch.json b/.vscode/launch.json index 768d17a8f09d98..ca66e559bd2554 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -15,7 +15,7 @@ "type": "msedge", "request": "launch", "url": "http://localhost:8888/wp-admin", - "runtimeExecutable": "/Applications/Vivaldi.app/Contents/MacOS/Vivaldi", + "runtimeExecutable": "/Applications/Vivaldi.app/Contents/MacOS/Vivaldi" }, { "type": "node", diff --git a/packages/core-data/src/reducer.js b/packages/core-data/src/reducer.js index 9748355fc5caf6..f6324b2719240f 100644 --- a/packages/core-data/src/reducer.js +++ b/packages/core-data/src/reducer.js @@ -472,6 +472,9 @@ export const entities = ( state = {}, action ) => { * @type {UndoManager} */ export function undoManager( state = createUndoManager() ) { + if ( window.__experimentalEnableSync ) { + // Replace the undo manager with the one provided by the sync provider. + } return state; } diff --git a/packages/sync/src/provider.ts b/packages/sync/src/provider.ts index 9d8a5139637ae7..fbd013e75b7704 100644 --- a/packages/sync/src/provider.ts +++ b/packages/sync/src/provider.ts @@ -6,6 +6,7 @@ import * as Y from 'yjs'; /** * Internal dependencies */ +import { UndoManager } from './undo-manager'; import type { ConnectDoc, ConnectDocResult, @@ -18,7 +19,7 @@ import type { } from './types'; interface EntityState { - undoManager: Y.UndoManager; + undoManager: UndoManager; destroy: () => void; ydoc: CRDTDoc; } @@ -83,7 +84,7 @@ export class SyncProvider { const connections = await this.connect( objectId, objectType, ydoc ); const entityId = this.getEntityId( objectType, objectId ); - const undoManager = new Y.UndoManager( ydoc.getMap( 'document' ), { + const yjsUndoManager = new Y.UndoManager( ydoc.getMap( 'document' ), { // Ensure we undo and redo one character at a time. captureTimeout: 0, // Ensure that we only scope the undo/redo to the current client, and Gutenberg origins. @@ -94,11 +95,12 @@ export class SyncProvider { ignoreRemoteMapChanges: true, } ); + const undoManager = new UndoManager( yjsUndoManager ); + const onDestroy = (): void => { connections.forEach( ( result ) => result.destroy() ); ydoc.off( 'update', onUpdate ); ydoc.destroy(); - undoManager.destroy(); this.entityStates.delete( entityId ); }; @@ -161,7 +163,7 @@ export class SyncProvider { public getUndoManager( objectType: ObjectType, objectId: ObjectID - ): Y.UndoManager | null { + ): UndoManager | null { const entityState = this.getEntityState( objectType, objectId ); return entityState ? entityState.undoManager : null; } diff --git a/packages/sync/src/undo-manager.ts b/packages/sync/src/undo-manager.ts new file mode 100644 index 00000000000000..63fc5f0f630b44 --- /dev/null +++ b/packages/sync/src/undo-manager.ts @@ -0,0 +1,100 @@ +/** + * External dependencies + */ +import type * as Y from 'yjs'; + +/** + * WordPress dependencies + */ +import type { + HistoryRecord, + UndoManager as WPUndoManager, +} from '@wordpress/undo-manager'; + +/** + * Internal dependencies + */ +import type { ObjectData } from './types'; + +/** + * Wrapper class that provides the WordPress UndoManager interface while using Y.UndoManager internally. + * This allows seamless integration between Yjs collaborative editing and WordPress undo/redo functionality. + */ +export class UndoManager implements WPUndoManager< ObjectData > { + private undoManager: Y.UndoManager; + + /** + * Constructor. + * + * @param undoManager The Y.UndoManager instance to wrap. + */ + constructor( undoManager: Y.UndoManager ) { + this.undoManager = undoManager; + } + + /** + * Record changes into the history. + * Since Yjs automatically tracks changes, this method translates the WordPress + * HistoryRecord format into Yjs operations. + * + * @param record A record of changes to record. + * @param isStaged Whether to immediately create an undo point or not. + */ + // eslint-disable-next-line @typescript-eslint/no-unused-vars + addRecord( record?: HistoryRecord< ObjectData >, isStaged = false ): void { + // This is a no-op for Yjs since it automatically tracks changes. + // If needed, we could implement custom logic to handle specific records. + } + + /** + * Undo the last recorded changes. + * + * @return The undone record or undefined if nothing to undo. + */ + undo(): HistoryRecord< ObjectData > | undefined { + if ( ! this.hasUndo() ) { + return undefined; + } + + // Perform the undo operation + this.undoManager.undo(); + + // ToDo: See if the undo operation can return a record from Yjs. + return []; + } + + /** + * Redo the last undone changes. + * + * @return The redone record or undefined if nothing to redo. + */ + redo(): HistoryRecord< ObjectData > | undefined { + if ( ! this.hasRedo() ) { + return undefined; + } + + // Perform the redo operation + this.undoManager.redo(); + + // ToDo: See if the redo operation can return a record from Yjs. + return []; + } + + /** + * Check if there are changes that can be undone. + * + * @return Whether there are changes to undo. + */ + hasUndo(): boolean { + return this.undoManager.canUndo(); + } + + /** + * Check if there are changes that can be redone. + * + * @return Whether there are changes to redo. + */ + hasRedo(): boolean { + return this.undoManager.canRedo(); + } +} From 9ef29f5781c4210114b6f9ea17fc43fd2eff6a62 Mon Sep 17 00:00:00 2001 From: chriszarate Date: Wed, 6 Aug 2025 09:28:56 -0400 Subject: [PATCH 19/54] Cleanup and separation of concerns --- packages/sync/src/provider.ts | 18 ++---------- packages/sync/src/undo-manager.ts | 46 +++++++++++++++++-------------- 2 files changed, 29 insertions(+), 35 deletions(-) diff --git a/packages/sync/src/provider.ts b/packages/sync/src/provider.ts index fbd013e75b7704..48b2130dbfdb45 100644 --- a/packages/sync/src/provider.ts +++ b/packages/sync/src/provider.ts @@ -19,8 +19,8 @@ import type { } from './types'; interface EntityState { - undoManager: UndoManager; destroy: () => void; + undoManager: UndoManager; ydoc: CRDTDoc; } @@ -79,24 +79,12 @@ export class SyncProvider { handleChanges: ( data: Partial< ObjectData > ) => void ): Promise< void > { const ydoc = new Y.Doc( { meta: new Map() } ); + const undoManager = new UndoManager( ydoc ); const objectId = syncConfig.getObjectId( initialData ); const objectType = syncConfig.objectType; const connections = await this.connect( objectId, objectType, ydoc ); const entityId = this.getEntityId( objectType, objectId ); - const yjsUndoManager = new Y.UndoManager( ydoc.getMap( 'document' ), { - // Ensure we undo and redo one character at a time. - captureTimeout: 0, - // Ensure that we only scope the undo/redo to the current client, and Gutenberg origins. - // ToDo: Keep an eye on this, as it needs to be battle tested. - trackedOrigins: new Set( [ 'gutenberg', ydoc.clientID ] ), - // This ensures that are able to improve the client specific undo/redo experience. - // This reduces the bugs we see, but it doesn't eliminate them entirely. - ignoreRemoteMapChanges: true, - } ); - - const undoManager = new UndoManager( yjsUndoManager ); - const onDestroy = (): void => { connections.forEach( ( result ) => result.destroy() ); ydoc.off( 'update', onUpdate ); @@ -116,9 +104,9 @@ export class SyncProvider { this.configs.set( objectType, syncConfig ); this.connections.set( entityId, connections ); this.entityStates.set( entityId, { + destroy: onDestroy, undoManager, ydoc, - destroy: onDestroy, } ); this.update( objectType, initialData, initialData, 'gutenberg' ); diff --git a/packages/sync/src/undo-manager.ts b/packages/sync/src/undo-manager.ts index 63fc5f0f630b44..8cc6c784232f8d 100644 --- a/packages/sync/src/undo-manager.ts +++ b/packages/sync/src/undo-manager.ts @@ -1,7 +1,7 @@ /** * External dependencies */ -import type * as Y from 'yjs'; +import * as Y from 'yjs'; /** * WordPress dependencies @@ -14,7 +14,7 @@ import type { /** * Internal dependencies */ -import type { ObjectData } from './types'; +import type { CRDTDoc, ObjectData } from './types'; /** * Wrapper class that provides the WordPress UndoManager interface while using Y.UndoManager internally. @@ -23,13 +23,17 @@ import type { ObjectData } from './types'; export class UndoManager implements WPUndoManager< ObjectData > { private undoManager: Y.UndoManager; - /** - * Constructor. - * - * @param undoManager The Y.UndoManager instance to wrap. - */ - constructor( undoManager: Y.UndoManager ) { - this.undoManager = undoManager; + public constructor( ydoc: CRDTDoc ) { + this.undoManager = new Y.UndoManager( ydoc.getMap( 'document' ), { + // Ensure we undo and redo one character at a time. + captureTimeout: 0, + // Ensure that we only scope the undo/redo to the current client, and Gutenberg origins. + // ToDo: Keep an eye on this, as it needs to be battle tested. + trackedOrigins: new Set( [ 'gutenberg', ydoc.clientID ] ), + // This ensures that are able to improve the client specific undo/redo experience. + // This reduces the bugs we see, but it doesn't eliminate them entirely. + ignoreRemoteMapChanges: true, + } ); } /** @@ -37,11 +41,13 @@ export class UndoManager implements WPUndoManager< ObjectData > { * Since Yjs automatically tracks changes, this method translates the WordPress * HistoryRecord format into Yjs operations. * - * @param record A record of changes to record. - * @param isStaged Whether to immediately create an undo point or not. + * @param _record A record of changes to record. + * @param _isStaged Whether to immediately create an undo point or not. */ - // eslint-disable-next-line @typescript-eslint/no-unused-vars - addRecord( record?: HistoryRecord< ObjectData >, isStaged = false ): void { + public addRecord( + _record?: HistoryRecord< ObjectData >, + _isStaged = false // eslint-disable-line @typescript-eslint/no-unused-vars + ): void { // This is a no-op for Yjs since it automatically tracks changes. // If needed, we could implement custom logic to handle specific records. } @@ -51,7 +57,7 @@ export class UndoManager implements WPUndoManager< ObjectData > { * * @return The undone record or undefined if nothing to undo. */ - undo(): HistoryRecord< ObjectData > | undefined { + public undo(): HistoryRecord< ObjectData > | undefined { if ( ! this.hasUndo() ) { return undefined; } @@ -59,7 +65,7 @@ export class UndoManager implements WPUndoManager< ObjectData > { // Perform the undo operation this.undoManager.undo(); - // ToDo: See if the undo operation can return a record from Yjs. + // @TODO See if the undo operation can return a record from Yjs. return []; } @@ -68,15 +74,15 @@ export class UndoManager implements WPUndoManager< ObjectData > { * * @return The redone record or undefined if nothing to redo. */ - redo(): HistoryRecord< ObjectData > | undefined { + public redo(): HistoryRecord< ObjectData > | undefined { if ( ! this.hasRedo() ) { - return undefined; + return; } // Perform the redo operation this.undoManager.redo(); - // ToDo: See if the redo operation can return a record from Yjs. + // @TODO See if the redo operation can return a record from Yjs. return []; } @@ -85,7 +91,7 @@ export class UndoManager implements WPUndoManager< ObjectData > { * * @return Whether there are changes to undo. */ - hasUndo(): boolean { + public hasUndo(): boolean { return this.undoManager.canUndo(); } @@ -94,7 +100,7 @@ export class UndoManager implements WPUndoManager< ObjectData > { * * @return Whether there are changes to redo. */ - hasRedo(): boolean { + public hasRedo(): boolean { return this.undoManager.canRedo(); } } From cdb3bc4c0ab2ca072546fad35dcc022a1728953d Mon Sep 17 00:00:00 2001 From: chriszarate Date: Wed, 6 Aug 2025 11:40:58 -0400 Subject: [PATCH 20/54] Rely on a single instance of UndoManager --- packages/core-data/src/entities.js | 1 + packages/core-data/src/private-selectors.ts | 3 +- packages/core-data/src/selectors.ts | 5 +-- packages/sync/src/provider.ts | 35 +++++++++++++-------- packages/sync/src/types.ts | 1 + 5 files changed, 29 insertions(+), 16 deletions(-) diff --git a/packages/core-data/src/entities.js b/packages/core-data/src/entities.js index 85f381ef5ef7fe..e378718c4fb904 100644 --- a/packages/core-data/src/entities.js +++ b/packages/core-data/src/entities.js @@ -715,6 +715,7 @@ async function loadPostTypeEntities() { fromCRDTDoc: defaultYdocTransformer, getObjectId: ( { id } ) => id, objectType: 'postType/' + postType.name, + supportsUndo: true, }, supportsPagination: true, getRevisionsUrl: ( parentId, revisionId ) => diff --git a/packages/core-data/src/private-selectors.ts b/packages/core-data/src/private-selectors.ts index 2544c0b3b5e67c..d3ae0eb2094fd6 100644 --- a/packages/core-data/src/private-selectors.ts +++ b/packages/core-data/src/private-selectors.ts @@ -9,6 +9,7 @@ import { createSelector, createRegistrySelector } from '@wordpress/data'; import { getDefaultTemplateId, getEntityRecord, type State } from './selectors'; import { STORE_NAME } from './name'; import { unlock } from './lock-unlock'; +import { getSyncProvider } from './sync'; type EntityRecordKey = string | number; @@ -21,7 +22,7 @@ type EntityRecordKey = string | number; * @return The undo manager. */ export function getUndoManager( state: State ) { - return state.undoManager; + return getSyncProvider().getUndoManager() ?? state.undoManager; } /** diff --git a/packages/core-data/src/selectors.ts b/packages/core-data/src/selectors.ts index 64ff340afd0c27..a4827b45095ce8 100644 --- a/packages/core-data/src/selectors.ts +++ b/packages/core-data/src/selectors.ts @@ -15,6 +15,7 @@ import { getQueriedTotalPages, } from './queried-data'; import { DEFAULT_ENTITY_KEY } from './entities'; +import { getUndoManager } from './private-selectors'; import { getNormalizedCommaSeparable, isRawAttribute, @@ -1048,7 +1049,7 @@ export function getRedoEdit( state: State ): Optional< any > { * @return Whether there is a previous edit or not. */ export function hasUndo( state: State ): boolean { - return state.undoManager.hasUndo(); + return getUndoManager( state ).hasUndo(); } /** @@ -1060,7 +1061,7 @@ export function hasUndo( state: State ): boolean { * @return Whether there is a next edit or not. */ export function hasRedo( state: State ): boolean { - return state.undoManager.hasRedo(); + return getUndoManager( state ).hasRedo(); } /** diff --git a/packages/sync/src/provider.ts b/packages/sync/src/provider.ts index 48b2130dbfdb45..04d9098515b15e 100644 --- a/packages/sync/src/provider.ts +++ b/packages/sync/src/provider.ts @@ -20,7 +20,6 @@ import type { interface EntityState { destroy: () => void; - undoManager: UndoManager; ydoc: CRDTDoc; } @@ -28,6 +27,20 @@ export class SyncProvider { private connectLocal: ConnectDoc | null; private connectRemote: ConnectDoc | null; + /** + * CAUTION: We currently store a single UndoManager instance under these + * assumptions: + * + * 1. Only entities loaded by the block editor support an undo manager. + * 2. Only one such entity is loaded at a time. + * 3. The entity's SyncConfig has `supportsUndo` set to true. + * + * If these assumptions fail, we will need to refactor the selectors provided + * by `@wordpress/core-data` (e.g., `getUndoManager`) to support multiple + * UndoManager instances by requiring the entity type and ID as parameters. + */ + private undoManager: UndoManager | null = null; + protected configs: Map< ObjectType, SyncConfig > = new Map(); protected connections: Map< EntityID, ConnectDocResult[] > = new Map(); protected entityStates: Map< EntityID, EntityState > = new Map(); @@ -79,7 +92,6 @@ export class SyncProvider { handleChanges: ( data: Partial< ObjectData > ) => void ): Promise< void > { const ydoc = new Y.Doc( { meta: new Map() } ); - const undoManager = new UndoManager( ydoc ); const objectId = syncConfig.getObjectId( initialData ); const objectType = syncConfig.objectType; const connections = await this.connect( objectId, objectType, ydoc ); @@ -101,11 +113,14 @@ export class SyncProvider { ydoc.on( 'update', onUpdate ); + if ( syncConfig.supportsUndo ) { + this.undoManager = new UndoManager( ydoc ); + } + this.configs.set( objectType, syncConfig ); this.connections.set( entityId, connections ); this.entityStates.set( entityId, { destroy: onDestroy, - undoManager, ydoc, } ); @@ -142,18 +157,12 @@ export class SyncProvider { } /** - * Get the undo manager for the given object type and object ID. + * Get the undo manager. * - * @param {ObjectType} objectType Object type. - * @param {ObjectID} objectId Object ID. - * @return {Y.UndoManager | null} The undo manager, or null if not found. + * @return {Y.UndoManager | null} The undo manager, or null if unsupported. */ - public getUndoManager( - objectType: ObjectType, - objectId: ObjectID - ): UndoManager | null { - const entityState = this.getEntityState( objectType, objectId ); - return entityState ? entityState.undoManager : null; + public getUndoManager(): UndoManager | null { + return this.undoManager; } /** diff --git a/packages/sync/src/types.ts b/packages/sync/src/types.ts index bca2d1364112e7..9a5fcbcf83bd0a 100644 --- a/packages/sync/src/types.ts +++ b/packages/sync/src/types.ts @@ -29,4 +29,5 @@ export type SyncConfig = { fromCRDTDoc: ( ydoc: Y.Doc ) => ObjectData; getObjectId: ( data: ObjectData ) => ObjectID; objectType: ObjectType; + supportsUndo?: boolean; }; From c9844b5277c7eab24ae3ae10b0dccf201eeb19dd Mon Sep 17 00:00:00 2001 From: chriszarate Date: Wed, 6 Aug 2025 11:42:58 -0400 Subject: [PATCH 21/54] Remove launch.json --- .vscode/launch.json | 38 -------------------------------------- 1 file changed, 38 deletions(-) delete mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json deleted file mode 100644 index ca66e559bd2554..00000000000000 --- a/.vscode/launch.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "version": "0.2.0", - "configurations": [ - { - "name": "Listen for Xdebug", - "type": "php", - "request": "launch", - "port": 9003, - "pathMappings": { - "/var/www/html/wp-content/plugins/gutenberg": "${workspaceFolder}" - } - }, - { - "name": "Debug Vivaldi", - "type": "msedge", - "request": "launch", - "url": "http://localhost:8888/wp-admin", - "runtimeExecutable": "/Applications/Vivaldi.app/Contents/MacOS/Vivaldi" - }, - { - "type": "node", - "request": "launch", - "name": "Debug current e2e test", - "program": "${workspaceFolder}/node_modules/@wordpress/scripts/bin/wp-scripts.js", - "args": [ - "test-e2e", - "--config=${workspaceFolder}/packages/e2e-tests/jest.config.js", - "--verbose=true", - "--runInBand", - "--watch", - "${file}" - ], - "console": "integratedTerminal", - "internalConsoleOptions": "neverOpen", - "trace": "all" - } - ] -} From c5ae8fbf0c6151c123b40785ee1dae311552e816 Mon Sep 17 00:00:00 2001 From: chriszarate Date: Wed, 6 Aug 2025 11:45:41 -0400 Subject: [PATCH 22/54] Remove feature flag check --- packages/core-data/src/reducer.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/core-data/src/reducer.js b/packages/core-data/src/reducer.js index f6324b2719240f..9748355fc5caf6 100644 --- a/packages/core-data/src/reducer.js +++ b/packages/core-data/src/reducer.js @@ -472,9 +472,6 @@ export const entities = ( state = {}, action ) => { * @type {UndoManager} */ export function undoManager( state = createUndoManager() ) { - if ( window.__experimentalEnableSync ) { - // Replace the undo manager with the one provided by the sync provider. - } return state; } From f8d0ac2d3c77f01fe3e6aaaf993b6b241fc18142 Mon Sep 17 00:00:00 2001 From: chriszarate Date: Wed, 6 Aug 2025 16:34:49 -0400 Subject: [PATCH 23/54] Add supportsAwareness: true to post entities --- packages/core-data/src/entities.js | 1 + packages/sync/src/types.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/packages/core-data/src/entities.js b/packages/core-data/src/entities.js index 85f381ef5ef7fe..fe3db7c1007512 100644 --- a/packages/core-data/src/entities.js +++ b/packages/core-data/src/entities.js @@ -715,6 +715,7 @@ async function loadPostTypeEntities() { fromCRDTDoc: defaultYdocTransformer, getObjectId: ( { id } ) => id, objectType: 'postType/' + postType.name, + supportsAwareness: true, }, supportsPagination: true, getRevisionsUrl: ( parentId, revisionId ) => diff --git a/packages/sync/src/types.ts b/packages/sync/src/types.ts index bca2d1364112e7..bfd10a3ea5922c 100644 --- a/packages/sync/src/types.ts +++ b/packages/sync/src/types.ts @@ -29,4 +29,5 @@ export type SyncConfig = { fromCRDTDoc: ( ydoc: Y.Doc ) => ObjectData; getObjectId: ( data: ObjectData ) => ObjectID; objectType: ObjectType; + supportsAwareness?: boolean; }; From 7132976829ba7642d696cf4a1ed23686bd1ce930 Mon Sep 17 00:00:00 2001 From: ingeniumed Date: Thu, 7 Aug 2025 10:48:27 +1000 Subject: [PATCH 24/54] Minor change in the return for undo and undo manger return type --- packages/sync/src/provider.ts | 2 +- packages/sync/src/undo-manager.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/sync/src/provider.ts b/packages/sync/src/provider.ts index 04d9098515b15e..1ee97b33065737 100644 --- a/packages/sync/src/provider.ts +++ b/packages/sync/src/provider.ts @@ -159,7 +159,7 @@ export class SyncProvider { /** * Get the undo manager. * - * @return {Y.UndoManager | null} The undo manager, or null if unsupported. + * @return {UndoManager | null} The undo manager, or null if unsupported. */ public getUndoManager(): UndoManager | null { return this.undoManager; diff --git a/packages/sync/src/undo-manager.ts b/packages/sync/src/undo-manager.ts index 8cc6c784232f8d..63c24234094272 100644 --- a/packages/sync/src/undo-manager.ts +++ b/packages/sync/src/undo-manager.ts @@ -59,7 +59,7 @@ export class UndoManager implements WPUndoManager< ObjectData > { */ public undo(): HistoryRecord< ObjectData > | undefined { if ( ! this.hasUndo() ) { - return undefined; + return; } // Perform the undo operation From 640177a045a8f39971659a80455cbb2dbb4abc5f Mon Sep 17 00:00:00 2001 From: ingeniumed Date: Thu, 7 Aug 2025 12:00:35 +1000 Subject: [PATCH 25/54] Fix the missing path in tsconfig for hooks --- packages/sync/tsconfig.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/sync/tsconfig.json b/packages/sync/tsconfig.json index f0a5cb0530d297..53e6a2b663d310 100644 --- a/packages/sync/tsconfig.json +++ b/packages/sync/tsconfig.json @@ -4,5 +4,5 @@ "compilerOptions": { "types": [ "node" ] }, - "references": [ { "path": "../url" } ] + "references": [ { "path": "../hooks" }, { "path": "../url" } ] } From 3d4e50ee6ec4b304a6f005a71dcd138f0f1c0890 Mon Sep 17 00:00:00 2001 From: chriszarate Date: Thu, 14 Aug 2025 16:33:45 -0600 Subject: [PATCH 26/54] Remove sync configs for non-synced entities --- packages/core-data/src/entities.js | 114 ----------------------------- 1 file changed, 114 deletions(-) diff --git a/packages/core-data/src/entities.js b/packages/core-data/src/entities.js index acdf314616b760..4d0c2be5bc985e 100644 --- a/packages/core-data/src/entities.js +++ b/packages/core-data/src/entities.js @@ -129,44 +129,6 @@ export const rootEntitiesConfig = [ // The entity doesn't support selecting multiple records. // The property is maintained for backward compatibility. plural: '__unstableBases', - syncConfig: { - applyChangesToDoc: ( doc, changes ) => { - const content = changes.content?.raw || changes.content; - const parsedYdoc = - typeof content === 'string' - ? parseContentYdoc( 'root/base', content ) - : null; // Note: always use the same 'postType' as this object's config.syncObjectType - if ( parsedYdoc !== null ) { - // parse content which contains a ydoc, and apply it to the current ydoc. The rest of the attributes can be ignored. - Y.transact( - doc, - () => { - // apply remote changes - Y.applyUpdate( - doc, - Y.encodeStateAsUpdate( parsedYdoc ) - ); - }, - 'applyChangesToDoc', - false - ); - } else { - // local changes happened. Apply the differences to the ydoc - const ycontent = doc.getMap( 'document' ); - Object.entries( changes ).forEach( ( [ key, value ] ) => { - if ( - ! filteredAttributes.has( key ) && - ! fun.equalityDeep( ycontent.get( key ), value ) - ) { - ycontent.set( key, value ); - } - } ); - } - }, - fromCRDTDoc: defaultYdocTransformer, - getObjectId: () => 'index', - objectType: 'root/base', - }, }, { label: __( 'Post Type' ), @@ -176,44 +138,6 @@ export const rootEntitiesConfig = [ baseURL: '/wp/v2/types', baseURLParams: { context: 'edit' }, plural: 'postTypes', - syncConfig: { - applyChangesToDoc: ( ydoc, changes ) => { - const content = changes.content?.raw || changes.content; - const parsedYdoc = - typeof content === 'string' - ? parseContentYdoc( 'root/postType', content ) - : null; // Note: always use the same 'postType' as this object's config.syncObjectType - if ( parsedYdoc !== null ) { - // parse content which contains a ydoc, and apply it to the current ydoc. The rest of the attributes can be ignored. - Y.transact( - ydoc, - () => { - // apply remote changes - Y.applyUpdate( - ydoc, - Y.encodeStateAsUpdate( parsedYdoc ) - ); - }, - 'applyChangesToDoc', - false - ); - } else { - // local changes happened. Apply the differences to the ydoc - const ycontent = ydoc.getMap( 'document' ); - Object.entries( changes ).forEach( ( [ key, value ] ) => { - if ( - ! filteredAttributes.has( key ) && - ! fun.equalityDeep( ycontent.get( key ), value ) - ) { - ycontent.set( key, value ); - } - } ); - } - }, - fromCRDTDoc: defaultYdocTransformer, - getObjectId: ( { id } ) => id, - objectType: 'root/postType', - }, }, { name: 'media', @@ -775,44 +699,6 @@ async function loadSiteEntity() { name: 'site', kind: 'root', baseURL: '/wp/v2/settings', - syncConfig: { - applyChangesToDoc: ( doc, changes ) => { - const content = changes.content?.raw || changes.content; - const parsedYdoc = - typeof content === 'string' - ? parseContentYdoc( 'root/site', content ) - : null; // Note: always use the same 'postType' as this object's config.syncObjectType - if ( parsedYdoc !== null ) { - // parse content which contains a ydoc, and apply it to the current ydoc. The rest of the attributes can be ignored. - Y.transact( - doc, - () => { - // apply remote changes - Y.applyUpdate( - doc, - Y.encodeStateAsUpdate( parsedYdoc ) - ); - }, - 'applyChangesToDoc', - false - ); - } else { - // local changes happened. Apply the differences to the ydoc - const ycontent = doc.getMap( 'document' ); - Object.entries( changes ).forEach( ( [ key, value ] ) => { - if ( - ! filteredAttributes.has( key ) && - ! fun.equalityDeep( ycontent.get( key ), value ) - ) { - ycontent.set( key, value ); - } - } ); - } - }, - fromCRDTDoc: defaultYdocTransformer, - getObjectId: () => 'index', - objectType: 'root/site', - }, meta: {}, }; From c490c8a2f4166ed83cd8e7ca326d098b31c6c4e1 Mon Sep 17 00:00:00 2001 From: chriszarate Date: Thu, 14 Aug 2025 16:35:22 -0600 Subject: [PATCH 27/54] Allow custom sync provider to override initial CRDT doc --- packages/core-data/src/entities.js | 522 +++++++++++------------------ packages/sync/src/provider.ts | 50 ++- packages/sync/src/types.ts | 3 +- 3 files changed, 242 insertions(+), 333 deletions(-) diff --git a/packages/core-data/src/entities.js b/packages/core-data/src/entities.js index 4d0c2be5bc985e..61ca424237ba2f 100644 --- a/packages/core-data/src/entities.js +++ b/packages/core-data/src/entities.js @@ -3,8 +3,7 @@ */ import { capitalCase, pascalCase } from 'change-case'; import { v4 as uuidv4 } from 'uuid'; -import * as string from 'lib0/string'; -import * as sha256 from 'lib0/hash/sha256'; + /** * WordPress dependencies */ @@ -19,82 +18,15 @@ import * as fun from 'lib0/function'; /** * Internal dependencies */ -import { getSyncProvider } from './sync'; export const DEFAULT_ENTITY_KEY = 'id'; const POST_RAW_ATTRIBUTES = [ 'title', 'excerpt', 'content' ]; -// @todo refactor `applyChangesToDoc` implementations this to have less repetition (there are -// multiple similar implementations) - -/** - * Similar to `parse`, but only reads the Yjs document if available. - * @param {string} postType - * @param {string} content - */ -export function parseContentYdoc( postType, content ) { - const newClientId = new Uint32Array( - sha256.digest( string.encodeUtf8( content ) ).buffer - ); - const syncProvider = getSyncProvider(); - - // It is important that this is a fresh document - don't use the document from the sync package! - const ydoc = new Y.Doc( { meta: new Map() } ); - - const knownUpdateGuids = new Set(); - ydoc.meta.set( 'knownRemoteUpdates', knownUpdateGuids ); - // Changing the Yjs clientid may lead to very weird bugs if done incorrectly. - // Please handle the following code-portion with great care! - const prevClientId = ydoc.clientID; - ydoc.clientID = newClientId; - const prevClock = ( ydoc.store.clients.get( newClientId ) || [ - { id: { clock: 0 } }, - ] )[ 0 ].id.clock; - const blocks = parse( content ); - syncProvider.configs.get( postType ).applyChangesToDoc( ydoc, { - blocks, - } ); - ydoc.clientID = prevClientId; - const newClock = ( ydoc.store.clients.get( newClientId ) ?? [ - { id: { clock: 0 } }, - ] )[ 0 ].id.clock; - if ( prevClock !== newClock ) { - // eslint-disable-next-line no-console - console.info( - '[Yjs Collab] Yjs document was updated to reflect changes to the HTML document.' - ); - } - return ydoc; -} - -// only sync what is necessary! -const filteredAttributes = new Set( [ - 'content', - 'selection', - 'excerpt', - 'date', - 'date_gmt', - 'format', - 'generated_slug', - 'link', - 'meta', - 'modified', - 'modified_gmt', - 'slug', - 'status', - 'sticky', - 'tags', - 'template', - '_links', - 'id', - 'password', - 'featured_media', -] ); - /** * @param {Y.Doc} ydoc + * @return {import('@wordpress/sync').ObjectData} The JSON representation of the document. */ -const defaultYdocTransformer = ( ydoc ) => { +const defaultFromCRDTDoc = ( ydoc ) => { const json = ydoc.getMap( 'document' ).toJSON(); if ( json.title?.raw ) { json.title = json.title.raw; @@ -355,6 +287,20 @@ function makeBlocksSerializable( blocks ) { * @return {Promise} Entities promise */ async function loadPostTypeEntities() { + const syncedProperties = new Set( [ + 'blocks', + 'content', + 'excerpt', + 'featured_media', + 'format', + 'generated_slug', + 'password', + 'slug', + 'sticky', + 'tags', + 'template', + ] ); + const postTypes = await apiFetch( { path: '/wp/v2/types?context=view', } ); @@ -388,267 +334,197 @@ async function loadPostTypeEntities() { * @param {Y.Doc} ydoc * @param {any} changes */ - applyChangesToDoc: ( ydoc, changes ) => { - const content = changes.content?.raw || changes.content; - const parsedYdoc = - typeof content === 'string' - ? parseContentYdoc( - 'postType/' + postType.name, - content - ) - : null; // Note: always use the same 'postType' as this object's config.syncObjectType - if ( parsedYdoc !== null ) { - // parse content which contains a ydoc, and apply it to the current ydoc. The rest of the attributes can be ignored. - Y.transact( - ydoc, - () => { - // apply remote changes - Y.applyUpdate( - ydoc, - Y.encodeStateAsUpdate( parsedYdoc ) + applyChangesToCRDTDoc: ( ydoc, changes ) => { + // local changes happened. Apply the differences to the ydoc + const ycontent = ydoc.getMap( 'document' ); + Object.entries( changes ).forEach( ( [ key, value ] ) => { + if ( typeof value !== 'function' ) { + if ( key === 'blocks' ) { + if ( ! serialisableBlocksCache.has( value ) ) { + serialisableBlocksCache.set( + value, + makeBlocksSerializable( value ) + ); + } + const blocks = + serialisableBlocksCache.get( value ); + // This is a rudimentary diff implementation similar to the y-prosemirror diffing + // approach. + // A better implementation would also diff the textual content and represent it + // using a Y.Text type. + // However, at this time it makes more sense to keep this algorithm generic to + // support all kinds of block types. + // Ideally, we ensure that block data structure have a consistent data format. + // E.g.: + // - textual content (using rich-text formatting?) may always be stored under `block.text` + // - local information that shouldn't be shared (e.g. clientId or isDragging) is stored under `block.private` + if ( + ! ycontent.has( key ) || + ycontent.get( key ) instanceof Array + ) { + // @todo remove the array check + ycontent.set( key, new Y.Array() ); + } + /** + * @type {Y.Array>} + */ + const yblocks = ycontent.get( key ); + const numOfCommonEntries = math.min( + blocks.length, + yblocks.length + ); + let left = 0; + let right = 0; + /** + * @param {any} gblock + * @param {Y.Map} yblock + */ + const blocksEqual = ( gblock, yblock ) => { + if ( yblock.toJSON ) { + yblock = yblock.toJSON(); + } + // we must not sync clientId, as this can't be generated consistenctly and + // hence will lead to merge conflicts. + const overwrites = { + innerBlocks: null, + clientId: null, + }; + const res = fun.equalityDeep( + Object.assign( {}, gblock, overwrites ), + Object.assign( {}, yblock, overwrites ) + ); + const inners = gblock.innerBlocks || []; + const yinners = yblock.innerBlocks || []; + return ( + res && + inners.length === yinners.length && + inners.every( ( block, i ) => + blocksEqual( block, yinners[ i ] ) + ) + ); + }; + // skip equal blocks from left + for ( + ; + left < numOfCommonEntries && + blocksEqual( + blocks[ left ], + yblocks.get( left ) + ); + left++ + ) { + /* nop */ + } + // skip equal blocks from right + for ( + ; + right < numOfCommonEntries - left && + blocksEqual( + blocks[ blocks.length - right - 1 ], + yblocks.get( + yblocks.length - right - 1 + ) + ); + right++ + ) { + /* nop */ + } + const numOfUpdatesNeeded = + numOfCommonEntries - left - right; + const numOfInsertionsNeeded = math.max( + 0, + blocks.length - yblocks.length + ); + const numOfDeletionsNeeded = math.max( + 0, + yblocks.length - blocks.length ); - }, - 'applyChangesToDoc', - false - ); - } else { - // local changes happened. Apply the differences to the ydoc - const ycontent = ydoc.getMap( 'document' ); - ydoc.transact( () => { - Object.entries( changes ).forEach( - ( [ key, value ] ) => { - if ( typeof value !== 'function' ) { - if ( key === 'blocks' ) { + // updates + for ( + let i = 0; + i < numOfUpdatesNeeded; + i++, left++ + ) { + const block = blocks[ left ]; + const yblock = yblocks.get( left ); + Object.entries( block ).forEach( + ( [ k, v ] ) => { if ( - ! serialisableBlocksCache.has( - value + ! fun.equalityDeep( + block[ k ], + yblock.get( k ) ) ) { - serialisableBlocksCache.set( - value, - makeBlocksSerializable( - value - ) - ); + yblock.set( k, v ); } - const blocks = - serialisableBlocksCache.get( - value - ); - // This is a rudimentary diff implementation similar to the y-prosemirror diffing - // approach. - // A better implementation would also diff the textual content and represent it - // using a Y.Text type. - // However, at this time it makes more sense to keep this algorithm generic to - // support all kinds of block types. - // Ideally, we ensure that block data structure have a consistent data format. - // E.g.: - // - textual content (using rich-text formatting?) may always be stored under `block.text` - // - local information that shouldn't be shared (e.g. clientId or isDragging) is stored under `block.private` - if ( - ! ycontent.has( key ) || - ycontent.get( key ) instanceof - Array - ) { - // @todo remove the array check - ycontent.set( - key, - new Y.Array() - ); - } - /** - * @type {Y.Array>} - */ - const yblocks = ycontent.get( key ); - const numOfCommonEntries = math.min( - blocks.length, - yblocks.length - ); - let left = 0; - let right = 0; - /** - * @param {any} gblock - * @param {Y.Map} yblock - */ - const blocksEqual = ( - gblock, - yblock - ) => { - if ( yblock.toJSON ) { - yblock = yblock.toJSON(); - } - // we must not sync clientId, as this can't be generated consistenctly and - // hence will lead to merge conflicts. - const overwrites = { - innerBlocks: null, - clientId: null, - }; - const res = fun.equalityDeep( - Object.assign( - {}, - gblock, - overwrites - ), - Object.assign( - {}, - yblock, - overwrites - ) - ); - const inners = - gblock.innerBlocks || []; - const yinners = - yblock.innerBlocks || []; - return ( - res && - inners.length === - yinners.length && - inners.every( - ( block, i ) => - blocksEqual( - block, - yinners[ i ] - ) - ) - ); - }; - // skip equal blocks from left - for ( - ; - left < numOfCommonEntries && - blocksEqual( - blocks[ left ], - yblocks.get( left ) - ); - left++ - ) { - /* nop */ - } - // skip equal blocks from right - for ( - ; - right < - numOfCommonEntries - left && - blocksEqual( - blocks[ - blocks.length - - right - - 1 - ], - yblocks.get( - yblocks.length - - right - - 1 - ) - ); - right++ - ) { - /* nop */ - } - const numOfUpdatesNeeded = - numOfCommonEntries - - left - - right; - const numOfInsertionsNeeded = - math.max( - 0, - blocks.length - - yblocks.length - ); - const numOfDeletionsNeeded = - math.max( - 0, - yblocks.length - - blocks.length - ); - // updates - for ( - let i = 0; - i < numOfUpdatesNeeded; - i++, left++ - ) { - const block = blocks[ left ]; - const yblock = - yblocks.get( left ); - Object.entries( block ).forEach( - ( [ k, v ] ) => { - if ( - ! fun.equalityDeep( - block[ k ], - yblock.get( k ) - ) - ) { - yblock.set( k, v ); - } - } - ); - yblock.forEach( ( _v, k ) => { - if ( - ! block.hasOwnProperty( - k - ) - ) { - yblock.delete( k ); - } - } ); - } - // deletes - yblocks.delete( - left, - numOfDeletionsNeeded - ); - // inserts - for ( - let i = 0; - i < numOfInsertionsNeeded; - i++, left++ - ) { - yblocks.insert( left, [ - new Y.Map( - Object.entries( - blocks[ left ] - ) - ), - ] ); - } - const knownClientIds = new Set(); - // remove duplicate clientids - for ( - let j = 0; - j < yblocks.length; - j++ - ) { - const yblock = yblocks.get( j ); - if ( - knownClientIds.has( - yblock.get( 'clientId' ) - ) - ) { - yblock.set( - 'clientId', - uuidv4() - ); - } - knownClientIds.add( - yblock.get( 'clientId' ) - ); - } - } else if ( - ! filteredAttributes.has( key ) && - ! fun.equalityDeep( - ycontent.get( key ), - value - ) - ) { - ycontent.set( key, value ); } + ); + yblock.forEach( ( _v, k ) => { + if ( ! block.hasOwnProperty( k ) ) { + yblock.delete( k ); + } + } ); + } + // deletes + yblocks.delete( left, numOfDeletionsNeeded ); + // inserts + for ( + let i = 0; + i < numOfInsertionsNeeded; + i++, left++ + ) { + yblocks.insert( left, [ + new Y.Map( + Object.entries( blocks[ left ] ) + ), + ] ); + } + const knownClientIds = new Set(); + // remove duplicate clientids + for ( let j = 0; j < yblocks.length; j++ ) { + const yblock = yblocks.get( j ); + if ( + knownClientIds.has( + yblock.get( 'clientId' ) + ) + ) { + yblock.set( 'clientId', uuidv4() ); } + knownClientIds.add( + yblock.get( 'clientId' ) + ); } - ); - }, 'gutenberg' ); - } + } else if ( + ! syncedProperties.has( key ) && + ! fun.equalityDeep( ycontent.get( key ), value ) + ) { + ycontent.set( key, value ); + } + } + } ); + }, + fromCRDTDoc: defaultFromCRDTDoc, + /** + * This initial object data represents the data that will be synced via + * the CRDT document, which may differ from the entity record. There may + * be properties that should not be synced, or properties that are + * derived from the record. + * + * @param {import('@wordpress/sync').ObjectData} record + * @return {import('@wordpress/sync').ObjectData} The initial data + */ + getInitialObjectData: ( record ) => { + // Mix in the parsed blocks into the record. Only allow properties in + // the synced properties set. + const content = record.content?.raw ?? record.content ?? ''; + const blocks = parse( content ); + + return Object.fromEntries( + Object.entries( { ...record, blocks } ).filter( + ( [ key ] ) => syncedProperties.has( key ) + ) + ); }, - fromCRDTDoc: defaultYdocTransformer, getObjectId: ( { id } ) => id, objectType: 'postType/' + postType.name, supportsAwareness: true, diff --git a/packages/sync/src/provider.ts b/packages/sync/src/provider.ts index 1ee97b33065737..1d960a5599e2f9 100644 --- a/packages/sync/src/provider.ts +++ b/packages/sync/src/provider.ts @@ -83,16 +83,16 @@ export class SyncProvider { * Fetch data from local database or remote source. * * @param {SyncConfig} syncConfig Sync configuration for the object type. - * @param {ObjectData} initialData Initial data to apply to the document. + * @param {ObjectData} record Record representing this object type. * @param {Function} handleChanges Callback to call when data changes. */ public async bootstrap( syncConfig: SyncConfig, - initialData: ObjectData, + record: ObjectData, handleChanges: ( data: Partial< ObjectData > ) => void ): Promise< void > { const ydoc = new Y.Doc( { meta: new Map() } ); - const objectId = syncConfig.getObjectId( initialData ); + const objectId = syncConfig.getObjectId( record ); const objectType = syncConfig.objectType; const connections = await this.connect( objectId, objectType, ydoc ); const entityId = this.getEntityId( objectType, objectId ); @@ -124,7 +124,19 @@ export class SyncProvider { ydoc, } ); - this.update( objectType, initialData, initialData, 'gutenberg' ); + // Get the initial data to be synced for this record. + const initialCRDTDoc = await this.getCRDTDoc( syncConfig, record ); + + // Create the initial document, possible from persisted doc. + Y.transact( + ydoc, + () => { + // apply remote changes + Y.applyUpdate( ydoc, Y.encodeStateAsUpdate( initialCRDTDoc ) ); + }, + 'syncProvider.bootstrap', + false + ); } /** @@ -156,6 +168,27 @@ export class SyncProvider { ); } + /** + * Get the CRDTDoc that represents the initial state of the object data. Custom + * sync providers can override this method to provide a custom initial state. + * + * @param {SyncConfig} syncConfig Sync configuration for the object type. + * @param {ObjectData} record Initial data to apply to the document. + */ + protected async getCRDTDoc( + syncConfig: SyncConfig, + record: ObjectData + ): Promise< CRDTDoc > { + // IMPORTANT: We use a new Yjs document so that the initial state can be + // applied to the "real" Yjs document as a singular update. + const initialStateDoc = new Y.Doc( { meta: new Map() } ); + + const initialData = syncConfig.getInitialObjectData( record ); + syncConfig.applyChangesToCRDTDoc( initialStateDoc, initialData ); + + return initialStateDoc; + } + /** * Get the undo manager. * @@ -179,18 +212,17 @@ export class SyncProvider { changes: Partial< ObjectData >, origin: string ): void { - const objectId = this.configs.get( objectType )?.getObjectId( record ); + const syncConfig = this.configs.get( objectType ); + const objectId = syncConfig?.getObjectId( record ); - if ( ! objectId ) { + if ( ! syncConfig || ! objectId ) { return; } const entityState = this.getEntityState( objectType, objectId ); entityState?.ydoc.transact( () => { - this.configs - .get( objectType ) - ?.applyChangesToDoc( entityState.ydoc, changes ); + syncConfig.applyChangesToCRDTDoc( entityState.ydoc, changes ); }, origin ); } diff --git a/packages/sync/src/types.ts b/packages/sync/src/types.ts index 1230b1b2e8c304..5b676821d67ac8 100644 --- a/packages/sync/src/types.ts +++ b/packages/sync/src/types.ts @@ -25,8 +25,9 @@ export type ConnectDoc = ( ) => Promise< ConnectDocResult >; export type SyncConfig = { - applyChangesToDoc: ( ydoc: Y.Doc, data: Partial< ObjectData > ) => void; + applyChangesToCRDTDoc: ( ydoc: Y.Doc, data: Partial< ObjectData > ) => void; fromCRDTDoc: ( ydoc: Y.Doc ) => ObjectData; + getInitialObjectData: ( record: ObjectData ) => ObjectData; getObjectId: ( data: ObjectData ) => ObjectID; objectType: ObjectType; supportsAwareness?: boolean; From c4d613d01be7c232c3d897b936733a1275d377b8 Mon Sep 17 00:00:00 2001 From: chriszarate Date: Fri, 15 Aug 2025 16:05:04 -0600 Subject: [PATCH 28/54] Rename method for clarity --- packages/sync/src/provider.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/sync/src/provider.ts b/packages/sync/src/provider.ts index 1d960a5599e2f9..8a457199556f75 100644 --- a/packages/sync/src/provider.ts +++ b/packages/sync/src/provider.ts @@ -125,7 +125,10 @@ export class SyncProvider { } ); // Get the initial data to be synced for this record. - const initialCRDTDoc = await this.getCRDTDoc( syncConfig, record ); + const initialCRDTDoc = await this.getInitialCRDTDoc( + syncConfig, + record + ); // Create the initial document, possible from persisted doc. Y.transact( @@ -175,7 +178,7 @@ export class SyncProvider { * @param {SyncConfig} syncConfig Sync configuration for the object type. * @param {ObjectData} record Initial data to apply to the document. */ - protected async getCRDTDoc( + protected async getInitialCRDTDoc( syncConfig: SyncConfig, record: ObjectData ): Promise< CRDTDoc > { From e51717d00654cb73467ee4df8516f500f5bbf98c Mon Sep 17 00:00:00 2001 From: chriszarate Date: Mon, 18 Aug 2025 13:16:20 -0600 Subject: [PATCH 29/54] Filter synced properties --- packages/core-data/src/entities.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/core-data/src/entities.js b/packages/core-data/src/entities.js index 61ca424237ba2f..51597a05584c7b 100644 --- a/packages/core-data/src/entities.js +++ b/packages/core-data/src/entities.js @@ -332,11 +332,15 @@ async function loadPostTypeEntities() { syncConfig: { /** * @param {Y.Doc} ydoc - * @param {any} changes + * @param {any} allChanges */ - applyChangesToCRDTDoc: ( ydoc, changes ) => { + applyChangesToCRDTDoc: ( ydoc, allChanges ) => { // local changes happened. Apply the differences to the ydoc const ycontent = ydoc.getMap( 'document' ); + const changes = Object.entries( allChanges ).filter( + ( [ key ] ) => syncedProperties.has( key ) + ); + Object.entries( changes ).forEach( ( [ key, value ] ) => { if ( typeof value !== 'function' ) { if ( key === 'blocks' ) { @@ -495,7 +499,6 @@ async function loadPostTypeEntities() { ); } } else if ( - ! syncedProperties.has( key ) && ! fun.equalityDeep( ycontent.get( key ), value ) ) { ycontent.set( key, value ); From 4c4fbc96122b8685c85abec8a2baf757faca088d Mon Sep 17 00:00:00 2001 From: chriszarate Date: Mon, 18 Aug 2025 13:19:53 -0600 Subject: [PATCH 30/54] Reform object --- packages/core-data/src/entities.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/core-data/src/entities.js b/packages/core-data/src/entities.js index 51597a05584c7b..0f5e2a9bf09dba 100644 --- a/packages/core-data/src/entities.js +++ b/packages/core-data/src/entities.js @@ -337,8 +337,10 @@ async function loadPostTypeEntities() { applyChangesToCRDTDoc: ( ydoc, allChanges ) => { // local changes happened. Apply the differences to the ydoc const ycontent = ydoc.getMap( 'document' ); - const changes = Object.entries( allChanges ).filter( - ( [ key ] ) => syncedProperties.has( key ) + const changes = Object.fromEntries( + Object.entries( allChanges ).filter( ( [ key ] ) => + syncedProperties.has( key ) + ) ); Object.entries( changes ).forEach( ( [ key, value ] ) => { From c9ffba1c39d95dc027057db05e007451e261d394 Mon Sep 17 00:00:00 2001 From: chriszarate Date: Mon, 18 Aug 2025 14:49:11 -0600 Subject: [PATCH 31/54] Remove content and excerpt from synced property list --- packages/core-data/src/entities.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/core-data/src/entities.js b/packages/core-data/src/entities.js index 0f5e2a9bf09dba..a29a25c60171ec 100644 --- a/packages/core-data/src/entities.js +++ b/packages/core-data/src/entities.js @@ -289,8 +289,6 @@ function makeBlocksSerializable( blocks ) { async function loadPostTypeEntities() { const syncedProperties = new Set( [ 'blocks', - 'content', - 'excerpt', 'featured_media', 'format', 'generated_slug', From 2f1213d0db31b20d15dc28eadec6d686b9a71dd2 Mon Sep 17 00:00:00 2001 From: chriszarate Date: Mon, 18 Aug 2025 18:19:44 -0600 Subject: [PATCH 32/54] Move merge functions to separate utils file --- packages/core-data/src/entities.js | 239 ++++---------------- packages/core-data/src/utils/crdt-blocks.ts | 183 +++++++++++++++ packages/core-data/src/utils/crdt.ts | 21 ++ packages/sync/src/provider.ts | 12 +- packages/sync/src/types.ts | 6 +- 5 files changed, 260 insertions(+), 201 deletions(-) create mode 100644 packages/core-data/src/utils/crdt-blocks.ts create mode 100644 packages/core-data/src/utils/crdt.ts diff --git a/packages/core-data/src/entities.js b/packages/core-data/src/entities.js index a29a25c60171ec..9afa1f18db27e2 100644 --- a/packages/core-data/src/entities.js +++ b/packages/core-data/src/entities.js @@ -2,22 +2,19 @@ * External dependencies */ import { capitalCase, pascalCase } from 'change-case'; -import { v4 as uuidv4 } from 'uuid'; /** * WordPress dependencies */ import apiFetch from '@wordpress/api-fetch'; import { __ } from '@wordpress/i18n'; -import { RichTextData } from '@wordpress/rich-text'; import { parse } from '@wordpress/blocks'; import { Y } from '@wordpress/sync'; -import * as math from 'lib0/math'; -import * as fun from 'lib0/function'; /** * Internal dependencies */ +import { mergeBlocks, mergePrimitiveValue } from './utils/crdt'; export const DEFAULT_ENTITY_KEY = 'id'; const POST_RAW_ATTRIBUTES = [ 'title', 'excerpt', 'content' ]; @@ -255,32 +252,6 @@ export const prePersistPostType = ( persistedRecord, edits ) => { return newEdits; }; -const serialisableBlocksCache = new WeakMap(); - -function makeBlockAttributesSerializable( attributes ) { - const newAttributes = { ...attributes }; - for ( const [ key, value ] of Object.entries( attributes ) ) { - if ( value instanceof RichTextData ) { - newAttributes[ key ] = value.valueOf(); - } - } - return newAttributes; -} - -function makeBlocksSerializable( blocks ) { - return blocks.map( ( block ) => { - const { innerBlocks, attributes, ...rest } = block; - delete rest.validationIssues; - delete rest.originalContent; - // delete rest.isValid - return { - ...rest, - attributes: makeBlockAttributesSerializable( attributes ), - innerBlocks: makeBlocksSerializable( innerBlocks ), - }; - } ); -} - /** * Returns the list of post type entities. * @@ -329,179 +300,55 @@ async function loadPostTypeEntities() { __unstable_rest_base: postType.rest_base, syncConfig: { /** - * @param {Y.Doc} ydoc - * @param {any} allChanges + * @param {Y.Doc} ydoc + * @param {any} changes + * @param {string} origin */ - applyChangesToCRDTDoc: ( ydoc, allChanges ) => { + applyChangesToCRDTDoc: ( ydoc, changes, origin ) => { // local changes happened. Apply the differences to the ydoc const ycontent = ydoc.getMap( 'document' ); - const changes = Object.fromEntries( - Object.entries( allChanges ).filter( ( [ key ] ) => - syncedProperties.has( key ) - ) + + const filteredEntries = Object.entries( changes ).filter( + ( [ key, value ] ) => + syncedProperties.has( key ) && + 'function' !== typeof value // cannot serialize function values ); - Object.entries( changes ).forEach( ( [ key, value ] ) => { - if ( typeof value !== 'function' ) { - if ( key === 'blocks' ) { - if ( ! serialisableBlocksCache.has( value ) ) { - serialisableBlocksCache.set( - value, - makeBlocksSerializable( value ) - ); - } - const blocks = - serialisableBlocksCache.get( value ); - // This is a rudimentary diff implementation similar to the y-prosemirror diffing - // approach. - // A better implementation would also diff the textual content and represent it - // using a Y.Text type. - // However, at this time it makes more sense to keep this algorithm generic to - // support all kinds of block types. - // Ideally, we ensure that block data structure have a consistent data format. - // E.g.: - // - textual content (using rich-text formatting?) may always be stored under `block.text` - // - local information that shouldn't be shared (e.g. clientId or isDragging) is stored under `block.private` - if ( - ! ycontent.has( key ) || - ycontent.get( key ) instanceof Array - ) { - // @todo remove the array check - ycontent.set( key, new Y.Array() ); - } - /** - * @type {Y.Array>} - */ - const yblocks = ycontent.get( key ); - const numOfCommonEntries = math.min( - blocks.length, - yblocks.length - ); - let left = 0; - let right = 0; - /** - * @param {any} gblock - * @param {Y.Map} yblock - */ - const blocksEqual = ( gblock, yblock ) => { - if ( yblock.toJSON ) { - yblock = yblock.toJSON(); - } - // we must not sync clientId, as this can't be generated consistenctly and - // hence will lead to merge conflicts. - const overwrites = { - innerBlocks: null, - clientId: null, - }; - const res = fun.equalityDeep( - Object.assign( {}, gblock, overwrites ), - Object.assign( {}, yblock, overwrites ) - ); - const inners = gblock.innerBlocks || []; - const yinners = yblock.innerBlocks || []; - return ( - res && - inners.length === yinners.length && - inners.every( ( block, i ) => - blocksEqual( block, yinners[ i ] ) - ) - ); - }; - // skip equal blocks from left - for ( - ; - left < numOfCommonEntries && - blocksEqual( - blocks[ left ], - yblocks.get( left ) - ); - left++ - ) { - /* nop */ - } - // skip equal blocks from right - for ( - ; - right < numOfCommonEntries - left && - blocksEqual( - blocks[ blocks.length - right - 1 ], - yblocks.get( - yblocks.length - right - 1 - ) - ); - right++ - ) { - /* nop */ + filteredEntries.forEach( ( [ key, newValue ] ) => { + const currentValue = ycontent.get( key ); + + // Return .get() result so that caller can operate on the data type + // without having to call .get() themselves. + function setValue( updatedValue ) { + ycontent.set( key, updatedValue ); + return ycontent.get( key ); + } + + switch ( key ) { + case 'blocks': { + let currentBlocks = currentValue; + if ( ! ( currentBlocks instanceof Y.Array ) ) { + currentBlocks = setValue( new Y.Array() ); // Initialize } - const numOfUpdatesNeeded = - numOfCommonEntries - left - right; - const numOfInsertionsNeeded = math.max( - 0, - blocks.length - yblocks.length - ); - const numOfDeletionsNeeded = math.max( - 0, - yblocks.length - blocks.length + + // Block[] from local changes or Y.Array< Y.Map > from peer. + const newBlocks = newValue ?? []; + + // Merge blocks does not need `setValue` because it has been + // called above and the result can be operated on directly. + mergeBlocks( currentBlocks, newBlocks, origin ); + break; + } + + // Add support for additional data types here. + + default: { + mergePrimitiveValue( + currentValue ?? undefined, + newValue ?? undefined, + setValue, + origin ); - // updates - for ( - let i = 0; - i < numOfUpdatesNeeded; - i++, left++ - ) { - const block = blocks[ left ]; - const yblock = yblocks.get( left ); - Object.entries( block ).forEach( - ( [ k, v ] ) => { - if ( - ! fun.equalityDeep( - block[ k ], - yblock.get( k ) - ) - ) { - yblock.set( k, v ); - } - } - ); - yblock.forEach( ( _v, k ) => { - if ( ! block.hasOwnProperty( k ) ) { - yblock.delete( k ); - } - } ); - } - // deletes - yblocks.delete( left, numOfDeletionsNeeded ); - // inserts - for ( - let i = 0; - i < numOfInsertionsNeeded; - i++, left++ - ) { - yblocks.insert( left, [ - new Y.Map( - Object.entries( blocks[ left ] ) - ), - ] ); - } - const knownClientIds = new Set(); - // remove duplicate clientids - for ( let j = 0; j < yblocks.length; j++ ) { - const yblock = yblocks.get( j ); - if ( - knownClientIds.has( - yblock.get( 'clientId' ) - ) - ) { - yblock.set( 'clientId', uuidv4() ); - } - knownClientIds.add( - yblock.get( 'clientId' ) - ); - } - } else if ( - ! fun.equalityDeep( ycontent.get( key ), value ) - ) { - ycontent.set( key, value ); } } } ); diff --git a/packages/core-data/src/utils/crdt-blocks.ts b/packages/core-data/src/utils/crdt-blocks.ts new file mode 100644 index 00000000000000..e0a8c89c15e07a --- /dev/null +++ b/packages/core-data/src/utils/crdt-blocks.ts @@ -0,0 +1,183 @@ +/** + * External dependencies + */ +import { v4 as uuidv4 } from 'uuid'; +import * as math from 'lib0/math'; +import * as fun from 'lib0/function'; + +/** + * WordPress dependencies + */ +import { RichTextData } from '@wordpress/rich-text'; +import { Y } from '@wordpress/sync'; + +interface BlockAttributes { + [ key: string ]: unknown; +} + +interface Block { + attributes: BlockAttributes; + clientId?: string; + innerBlocks: Block[]; + originalContent?: string; // unserializable + validationIssues?: string[]; // unserializable +} + +type Foo = Y.Map< Block[ keyof Block ] >; + +const serializableBlocksCache = new WeakMap< WeakKey, Block[] >(); + +function makeBlockAttributesSerializable( + attributes: BlockAttributes +): BlockAttributes { + const newAttributes = { ...attributes }; + for ( const [ key, value ] of Object.entries( attributes ) ) { + if ( value instanceof RichTextData ) { + newAttributes[ key ] = value.valueOf(); + } + } + return newAttributes; +} + +function makeBlocksSerializable( blocks: Block[] | Y.Array< Foo > ): Block[] { + return blocks.map( ( block: Block | Foo ) => { + const blockAsJson = block instanceof Y.Map ? block.toJSON() : block; + const { innerBlocks, attributes, ...rest } = blockAsJson; + delete rest.validationIssues; + delete rest.originalContent; + // delete rest.isValid + return { + ...rest, + attributes: makeBlockAttributesSerializable( attributes ), + innerBlocks: makeBlocksSerializable( innerBlocks ), + }; + } ); +} + +/** + * @param {any} gblock + * @param {Y.Map} yblock + */ +function areBlocksEqual( gblock: Block, yblock: Foo ): boolean { + const yblockAsJson = yblock.toJSON(); + + // we must not sync clientId, as this can't be generated consistenctly and + // hence will lead to merge conflicts. + const overwrites = { + innerBlocks: null, + clientId: null, + }; + const res = fun.equalityDeep( + Object.assign( {}, gblock, overwrites ), + Object.assign( {}, yblock, overwrites ) + ); + const inners = gblock.innerBlocks || []; + const yinners = yblockAsJson.innerBlocks || []; + return ( + res && + inners.length === yinners.length && + inners.every( ( block: Block, i: number ) => + areBlocksEqual( block, yinners[ i ] ) + ) + ); +} + +export function mergeBlocks( + yblocks: Y.Array< Foo >, + newValue: Block[] | Y.Array< Foo >, + _origin: string // eslint-disable-line @typescript-eslint/no-unused-vars +): void { + // Ensure we are working with serializable block data. + if ( ! serializableBlocksCache.has( newValue ) ) { + serializableBlocksCache.set( + newValue, + makeBlocksSerializable( newValue ) + ); + } + const blocks = serializableBlocksCache.get( newValue ) ?? []; + + // This is a rudimentary diff implementation similar to the y-prosemirror diffing + // approach. + // A better implementation would also diff the textual content and represent it + // using a Y.Text type. + // However, at this time it makes more sense to keep this algorithm generic to + // support all kinds of block types. + // Ideally, we ensure that block data structure have a consistent data format. + // E.g.: + // - textual content (using rich-text formatting?) may always be stored under `block.text` + // - local information that shouldn't be shared (e.g. clientId or isDragging) is stored under `block.private` + + const numOfCommonEntries = math.min( blocks.length ?? 0, yblocks.length ); + + let left = 0; + let right = 0; + + // skip equal blocks from left + for ( + ; + left < numOfCommonEntries && + areBlocksEqual( blocks[ left ], yblocks.get( left ) ); + left++ + ) { + /* nop */ + } + + // skip equal blocks from right + for ( + ; + right < numOfCommonEntries - left && + areBlocksEqual( + blocks[ blocks.length - right - 1 ], + yblocks.get( yblocks.length - right - 1 ) + ); + right++ + ) { + /* nop */ + } + + const numOfUpdatesNeeded = numOfCommonEntries - left - right; + const numOfInsertionsNeeded = math.max( 0, blocks.length - yblocks.length ); + const numOfDeletionsNeeded = math.max( 0, yblocks.length - blocks.length ); + + // updates + for ( let i = 0; i < numOfUpdatesNeeded; i++, left++ ) { + const block = blocks[ left ]; + const yblock = yblocks.get( left ); + Object.entries( block ).forEach( ( [ k, v ] ) => { + if ( ! fun.equalityDeep( block[ k ], yblock.get( k ) ) ) { + yblock.set( k, v ); + } + } ); + yblock.forEach( ( _v, k ) => { + if ( ! block.hasOwnProperty( k ) ) { + yblock.delete( k ); + } + } ); + } + + // deletes + yblocks.delete( left, numOfDeletionsNeeded ); + + // inserts + for ( let i = 0; i < numOfInsertionsNeeded; i++, left++ ) { + yblocks.insert( left, [ + new Y.Map< Block[ keyof Block ] >( + Object.entries( blocks[ left ] ) + ), + ] ); + } + + // remove duplicate clientids + const knownClientIds = new Set< string >(); + for ( let j = 0; j < yblocks.length; j++ ) { + const yblock: Y.Map< Block[ keyof Block ] > = yblocks.get( j ); + + let clientId: string = yblock.get( 'clientId' ) as string; + + if ( knownClientIds.has( clientId ) ) { + clientId = uuidv4(); + yblock.set( 'clientId', clientId ); + } + knownClientIds.add( clientId ); + } +} diff --git a/packages/core-data/src/utils/crdt.ts b/packages/core-data/src/utils/crdt.ts new file mode 100644 index 00000000000000..921eeed9f73e19 --- /dev/null +++ b/packages/core-data/src/utils/crdt.ts @@ -0,0 +1,21 @@ +/** + * External dependencies + */ +import * as fun from 'lib0/function'; + +export { mergeBlocks } from './crdt-blocks'; + +export type SetValueFunction< ValueType = unknown > = ( + value: ValueType +) => ValueType; + +export function mergePrimitiveValue< ValueType = unknown >( + currentValue: ValueType, + newValue: ValueType, + setValue: SetValueFunction< ValueType >, + _origin: string // eslint-disable-line @typescript-eslint/no-unused-vars +): void { + if ( ! fun.equalityDeep( currentValue, newValue ) ) { + setValue( newValue ); + } +} diff --git a/packages/sync/src/provider.ts b/packages/sync/src/provider.ts index 8a457199556f75..69297d4c5b25e1 100644 --- a/packages/sync/src/provider.ts +++ b/packages/sync/src/provider.ts @@ -187,7 +187,11 @@ export class SyncProvider { const initialStateDoc = new Y.Doc( { meta: new Map() } ); const initialData = syncConfig.getInitialObjectData( record ); - syncConfig.applyChangesToCRDTDoc( initialStateDoc, initialData ); + syncConfig.applyChangesToCRDTDoc( + initialStateDoc, + initialData, + 'syncProvider.getInitialCRDTDoc' + ); return initialStateDoc; } @@ -222,10 +226,10 @@ export class SyncProvider { return; } - const entityState = this.getEntityState( objectType, objectId ); + const ydoc = this.getEntityState( objectType, objectId )?.ydoc; - entityState?.ydoc.transact( () => { - syncConfig.applyChangesToCRDTDoc( entityState.ydoc, changes ); + ydoc?.transact( () => { + syncConfig.applyChangesToCRDTDoc( ydoc, changes, origin ); }, origin ); } diff --git a/packages/sync/src/types.ts b/packages/sync/src/types.ts index 5b676821d67ac8..7561c8afe1671a 100644 --- a/packages/sync/src/types.ts +++ b/packages/sync/src/types.ts @@ -25,7 +25,11 @@ export type ConnectDoc = ( ) => Promise< ConnectDocResult >; export type SyncConfig = { - applyChangesToCRDTDoc: ( ydoc: Y.Doc, data: Partial< ObjectData > ) => void; + applyChangesToCRDTDoc: ( + ydoc: Y.Doc, + data: Partial< ObjectData >, + origin: string + ) => void; fromCRDTDoc: ( ydoc: Y.Doc ) => ObjectData; getInitialObjectData: ( record: ObjectData ) => ObjectData; getObjectId: ( data: ObjectData ) => ObjectID; From 2bcce3012de1a7e393d0185c1556442fe9243eaa Mon Sep 17 00:00:00 2001 From: chriszarate Date: Mon, 18 Aug 2025 18:26:05 -0600 Subject: [PATCH 33/54] Rename Foo type and add comment --- packages/core-data/src/utils/crdt-blocks.ts | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/packages/core-data/src/utils/crdt-blocks.ts b/packages/core-data/src/utils/crdt-blocks.ts index e0a8c89c15e07a..326deb8768d96e 100644 --- a/packages/core-data/src/utils/crdt-blocks.ts +++ b/packages/core-data/src/utils/crdt-blocks.ts @@ -23,7 +23,11 @@ interface Block { validationIssues?: string[]; // unserializable } -type Foo = Y.Map< Block[ keyof Block ] >; +// The Y.Map type is not easy to work with. The generic type it accepts represents +// the possible values of the map, which are varied in our case. This type is +// accurate, but will require aggressive type narrowing when the map values are +// accessed -- or type casting with `as`. +type YBlock = Y.Map< Block[ keyof Block ] >; const serializableBlocksCache = new WeakMap< WeakKey, Block[] >(); @@ -39,8 +43,10 @@ function makeBlockAttributesSerializable( return newAttributes; } -function makeBlocksSerializable( blocks: Block[] | Y.Array< Foo > ): Block[] { - return blocks.map( ( block: Block | Foo ) => { +function makeBlocksSerializable( + blocks: Block[] | Y.Array< YBlock > +): Block[] { + return blocks.map( ( block: Block | YBlock ) => { const blockAsJson = block instanceof Y.Map ? block.toJSON() : block; const { innerBlocks, attributes, ...rest } = blockAsJson; delete rest.validationIssues; @@ -58,7 +64,7 @@ function makeBlocksSerializable( blocks: Block[] | Y.Array< Foo > ): Block[] { * @param {any} gblock * @param {Y.Map} yblock */ -function areBlocksEqual( gblock: Block, yblock: Foo ): boolean { +function areBlocksEqual( gblock: Block, yblock: YBlock ): boolean { const yblockAsJson = yblock.toJSON(); // we must not sync clientId, as this can't be generated consistenctly and @@ -83,8 +89,8 @@ function areBlocksEqual( gblock: Block, yblock: Foo ): boolean { } export function mergeBlocks( - yblocks: Y.Array< Foo >, - newValue: Block[] | Y.Array< Foo >, + yblocks: Y.Array< YBlock >, + newValue: Block[] | Y.Array< YBlock >, _origin: string // eslint-disable-line @typescript-eslint/no-unused-vars ): void { // Ensure we are working with serializable block data. From 74ed47b71b9f921952fc2eae848b48d78b94e2e9 Mon Sep 17 00:00:00 2001 From: chriszarate Date: Wed, 20 Aug 2025 21:02:11 -0600 Subject: [PATCH 34/54] Sync post title --- packages/core-data/src/entities.js | 39 +++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/packages/core-data/src/entities.js b/packages/core-data/src/entities.js index 9afa1f18db27e2..115465933f89dd 100644 --- a/packages/core-data/src/entities.js +++ b/packages/core-data/src/entities.js @@ -24,11 +24,7 @@ const POST_RAW_ATTRIBUTES = [ 'title', 'excerpt', 'content' ]; * @return {import('@wordpress/sync').ObjectData} The JSON representation of the document. */ const defaultFromCRDTDoc = ( ydoc ) => { - const json = ydoc.getMap( 'document' ).toJSON(); - if ( json.title?.raw ) { - json.title = json.title.raw; - } - return json; + return ydoc.getMap( 'document' ).toJSON(); }; export const rootEntitiesConfig = [ @@ -268,6 +264,7 @@ async function loadPostTypeEntities() { 'sticky', 'tags', 'template', + 'title', ] ); const postTypes = await apiFetch( { @@ -324,6 +321,16 @@ async function loadPostTypeEntities() { return ycontent.get( key ); } + // Set primitive a value (strings, numbers, booleans). + function setPrimitiveValue( primitiveValue ) { + mergePrimitiveValue( + currentValue ?? undefined, + primitiveValue ?? undefined, + setValue, + origin + ); + } + switch ( key ) { case 'blocks': { let currentBlocks = currentValue; @@ -340,15 +347,25 @@ async function loadPostTypeEntities() { break; } + case 'title': { + // Copy logic from prePersistPostType to ensure that the "Auto + // Draft" template title is not synced. + let rawNewValue = newValue?.raw ?? newValue; + if ( + ! currentValue && + 'Auto Draft' === rawNewValue + ) { + rawNewValue = ''; + } + + setPrimitiveValue( rawNewValue ); + break; + } + // Add support for additional data types here. default: { - mergePrimitiveValue( - currentValue ?? undefined, - newValue ?? undefined, - setValue, - origin - ); + setPrimitiveValue( newValue ); } } } ); From 6bbaa958196f502128a9e1708164faad7d034a35 Mon Sep 17 00:00:00 2001 From: ingeniumed Date: Thu, 21 Aug 2025 20:04:31 +1000 Subject: [PATCH 35/54] Improve block support --- packages/core-data/src/utils/crdt-blocks.ts | 36 +++++++++++++++++++-- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/packages/core-data/src/utils/crdt-blocks.ts b/packages/core-data/src/utils/crdt-blocks.ts index 326deb8768d96e..844a1a31bcf3a6 100644 --- a/packages/core-data/src/utils/crdt-blocks.ts +++ b/packages/core-data/src/utils/crdt-blocks.ts @@ -21,6 +21,7 @@ interface Block { innerBlocks: Block[]; originalContent?: string; // unserializable validationIssues?: string[]; // unserializable + name: string; } // The Y.Map type is not easy to work with. The generic type it accepts represents @@ -48,12 +49,13 @@ function makeBlocksSerializable( ): Block[] { return blocks.map( ( block: Block | YBlock ) => { const blockAsJson = block instanceof Y.Map ? block.toJSON() : block; - const { innerBlocks, attributes, ...rest } = blockAsJson; + const { name, innerBlocks, attributes, ...rest } = blockAsJson; delete rest.validationIssues; delete rest.originalContent; // delete rest.isValid return { ...rest, + name, attributes: makeBlockAttributesSerializable( attributes ), innerBlocks: makeBlocksSerializable( innerBlocks ), }; @@ -67,7 +69,7 @@ function makeBlocksSerializable( function areBlocksEqual( gblock: Block, yblock: YBlock ): boolean { const yblockAsJson = yblock.toJSON(); - // we must not sync clientId, as this can't be generated consistenctly and + // we must not sync clientId, as this can't be generated consistently and // hence will lead to merge conflicts. const overwrites = { innerBlocks: null, @@ -100,7 +102,12 @@ export function mergeBlocks( makeBlocksSerializable( newValue ) ); } - const blocks = serializableBlocksCache.get( newValue ) ?? []; + const unfilteredBlocks = serializableBlocksCache.get( newValue ) ?? []; + + // Ensure we skip blocks that we don't want to sync at the moment + const blocks = unfilteredBlocks.filter( ( block ) => + shouldBlockBeSynced( block ) + ); // This is a rudimentary diff implementation similar to the y-prosemirror diffing // approach. @@ -187,3 +194,26 @@ export function mergeBlocks( knownClientIds.add( clientId ); } } + +/** + * Determine if a block should be synced. + * + * Ex: A gallery block should not be synced until the images have been + * uploaded to WordPress, and their url is available. Before that, + * it's not possible to access the blobs on a client as those are + * local. + * + * @param block The block to check. + * @return True if the block should be synced, false otherwise. + */ +function shouldBlockBeSynced( block: Block ): boolean { + switch ( block.name ) { + case 'core/gallery': + return ! block.innerBlocks.some( + ( innerBlock ) => + innerBlock.attributes && innerBlock.attributes.blob + ); + default: + return true; + } +} From a78f85fd9fa3ffd6405a7120c11b0d0cbc966d79 Mon Sep 17 00:00:00 2001 From: chriszarate Date: Thu, 21 Aug 2025 16:12:14 -0600 Subject: [PATCH 36/54] Move applyChangesToDoc core code to utils/crdt --- packages/core-data/src/entities.js | 81 ++++--------------- packages/core-data/src/utils/crdt-blocks.ts | 4 +- packages/core-data/src/utils/crdt.ts | 86 +++++++++++++++++++-- 3 files changed, 94 insertions(+), 77 deletions(-) diff --git a/packages/core-data/src/entities.js b/packages/core-data/src/entities.js index 115465933f89dd..4999feae07d61e 100644 --- a/packages/core-data/src/entities.js +++ b/packages/core-data/src/entities.js @@ -9,12 +9,11 @@ import { capitalCase, pascalCase } from 'change-case'; import apiFetch from '@wordpress/api-fetch'; import { __ } from '@wordpress/i18n'; import { parse } from '@wordpress/blocks'; -import { Y } from '@wordpress/sync'; /** * Internal dependencies */ -import { mergeBlocks, mergePrimitiveValue } from './utils/crdt'; +import { defaultApplyChangesToCRDTDoc } from './utils/crdt'; export const DEFAULT_ENTITY_KEY = 'id'; const POST_RAW_ATTRIBUTES = [ 'title', 'excerpt', 'content' ]; @@ -298,77 +297,23 @@ async function loadPostTypeEntities() { syncConfig: { /** * @param {Y.Doc} ydoc - * @param {any} changes + * @param {Object} changes * @param {string} origin */ applyChangesToCRDTDoc: ( ydoc, changes, origin ) => { - // local changes happened. Apply the differences to the ydoc - const ycontent = ydoc.getMap( 'document' ); - - const filteredEntries = Object.entries( changes ).filter( - ( [ key, value ] ) => - syncedProperties.has( key ) && - 'function' !== typeof value // cannot serialize function values + const filteredChanges = Object.fromEntries( + Object.entries( changes ).filter( + ( [ key, value ] ) => + syncedProperties.has( key ) && + 'function' !== typeof value // cannot serialize function values + ) ); - filteredEntries.forEach( ( [ key, newValue ] ) => { - const currentValue = ycontent.get( key ); - - // Return .get() result so that caller can operate on the data type - // without having to call .get() themselves. - function setValue( updatedValue ) { - ycontent.set( key, updatedValue ); - return ycontent.get( key ); - } - - // Set primitive a value (strings, numbers, booleans). - function setPrimitiveValue( primitiveValue ) { - mergePrimitiveValue( - currentValue ?? undefined, - primitiveValue ?? undefined, - setValue, - origin - ); - } - - switch ( key ) { - case 'blocks': { - let currentBlocks = currentValue; - if ( ! ( currentBlocks instanceof Y.Array ) ) { - currentBlocks = setValue( new Y.Array() ); // Initialize - } - - // Block[] from local changes or Y.Array< Y.Map > from peer. - const newBlocks = newValue ?? []; - - // Merge blocks does not need `setValue` because it has been - // called above and the result can be operated on directly. - mergeBlocks( currentBlocks, newBlocks, origin ); - break; - } - - case 'title': { - // Copy logic from prePersistPostType to ensure that the "Auto - // Draft" template title is not synced. - let rawNewValue = newValue?.raw ?? newValue; - if ( - ! currentValue && - 'Auto Draft' === rawNewValue - ) { - rawNewValue = ''; - } - - setPrimitiveValue( rawNewValue ); - break; - } - - // Add support for additional data types here. - - default: { - setPrimitiveValue( newValue ); - } - } - } ); + defaultApplyChangesToCRDTDoc( + ydoc, + filteredChanges, + origin + ); }, fromCRDTDoc: defaultFromCRDTDoc, /** diff --git a/packages/core-data/src/utils/crdt-blocks.ts b/packages/core-data/src/utils/crdt-blocks.ts index 326deb8768d96e..adadafd07addd1 100644 --- a/packages/core-data/src/utils/crdt-blocks.ts +++ b/packages/core-data/src/utils/crdt-blocks.ts @@ -15,7 +15,7 @@ interface BlockAttributes { [ key: string ]: unknown; } -interface Block { +export interface Block { attributes: BlockAttributes; clientId?: string; innerBlocks: Block[]; @@ -27,7 +27,7 @@ interface Block { // the possible values of the map, which are varied in our case. This type is // accurate, but will require aggressive type narrowing when the map values are // accessed -- or type casting with `as`. -type YBlock = Y.Map< Block[ keyof Block ] >; +export type YBlock = Y.Map< Block[ keyof Block ] >; const serializableBlocksCache = new WeakMap< WeakKey, Block[] >(); diff --git a/packages/core-data/src/utils/crdt.ts b/packages/core-data/src/utils/crdt.ts index 921eeed9f73e19..5cfb047a16b310 100644 --- a/packages/core-data/src/utils/crdt.ts +++ b/packages/core-data/src/utils/crdt.ts @@ -3,17 +3,89 @@ */ import * as fun from 'lib0/function'; -export { mergeBlocks } from './crdt-blocks'; +/** + * WordPress dependencies + */ +import { type CRDTDoc, Y } from '@wordpress/sync'; + +/** + * Internal dependencies + */ +import { mergeBlocks, type Block, type YBlock } from './crdt-blocks'; + +type PrimitiveValue = string | number | boolean | null | undefined; + +interface PostChanges { + blocks?: Y.Array< YBlock > | Block[]; + title?: string | { raw: string }; +} + +export function defaultApplyChangesToCRDTDoc( + ydoc: CRDTDoc, + changes: PostChanges, + origin: string +): void { + const ymap = ydoc.getMap( 'document' ); -export type SetValueFunction< ValueType = unknown > = ( - value: ValueType -) => ValueType; + Object.entries( changes ).forEach( ( [ key, newValue ] ) => { + // Return .get() result so that caller can operate on the data type + // without having to call .get() themselves. + function setValue< T = unknown >( updatedValue: T ): T { + ymap.set( key, updatedValue ); + return ymap.get( key ) as T; + } + + switch ( key ) { + case 'blocks': { + let currentBlocks = ymap.get( + 'blocks' + ) as PostChanges[ 'blocks' ]; + + if ( ! ( currentBlocks instanceof Y.Array ) ) { + currentBlocks = setValue< Y.Array< YBlock > >( + new Y.Array() + ); // Initialize + } + + // Block[] from local changes or Y.Array< Y.Map > from peer. + const newBlocks = newValue ?? []; + + // Merge blocks does not need `setValue` because it has been + // called above and the result can be operated on directly. + mergeBlocks( currentBlocks, newBlocks, origin ); + break; + } + + case 'title': { + const currentValue = ymap.get( + 'title' + ) as PostChanges[ 'title' ]; + + // Copy logic from prePersistPostType to ensure that the "Auto + // Draft" template title is not synced. + let rawNewValue = newValue?.raw ?? newValue; + if ( ! currentValue && 'Auto Draft' === rawNewValue ) { + rawNewValue = ''; + } + + mergePrimitiveValue( currentValue, rawNewValue, setValue ); + break; + } + + // Add support for additional data types here. + + default: { + const currentValue = ymap.get( key ); + mergePrimitiveValue( currentValue, newValue, setValue ); + } + } + } ); +} -export function mergePrimitiveValue< ValueType = unknown >( +export function mergePrimitiveValue< ValueType extends PrimitiveValue >( currentValue: ValueType, newValue: ValueType, - setValue: SetValueFunction< ValueType >, - _origin: string // eslint-disable-line @typescript-eslint/no-unused-vars + setValue: ( value: ValueType ) => ValueType ): void { if ( ! fun.equalityDeep( currentValue, newValue ) ) { setValue( newValue ); From 8e268c56e1d0ba7d407d31dcd7e514582300675d Mon Sep 17 00:00:00 2001 From: chriszarate Date: Thu, 21 Aug 2025 16:38:46 -0600 Subject: [PATCH 37/54] Add post type support for collaborative editing --- lib/experimental/synchronization.php | 17 +++++++++++++++++ packages/core-data/src/entities.js | 7 +++++-- packages/core-data/src/resolvers.js | 2 +- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/lib/experimental/synchronization.php b/lib/experimental/synchronization.php index 87f13a7c685a59..029d47c8a365a9 100644 --- a/lib/experimental/synchronization.php +++ b/lib/experimental/synchronization.php @@ -22,3 +22,20 @@ function gutenberg_rest_api_init_collaborative_editing() { wp_add_inline_script( 'wp-sync', 'window.__experimentalCollaborativeEditingSecret = "' . $collaborative_editing_secret . '";', 'before' ); } add_action( 'admin_init', 'gutenberg_rest_api_init_collaborative_editing' ); + +/** + * Add support for collaborative editing to a some built-in post types. + */ +function gutenberg_add_collaborative_editing_post_type_support() { + $gutenberg_experiments = get_option( 'gutenberg-experiments' ); + if ( ! $gutenberg_experiments || ! array_key_exists( 'gutenberg-sync-collaboration', $gutenberg_experiments ) ) { + return; + } + + foreach ( array( 'page', 'post' ) as $post_type ) { + if ( post_type_exists( $post_type ) ) { + add_post_type_support( $post_type, 'collaborative-editing' ); + } + } +} +add_action( 'init', 'gutenberg_add_collaborative_editing_post_type_support', 10, 0 ); diff --git a/packages/core-data/src/entities.js b/packages/core-data/src/entities.js index 4999feae07d61e..7793728f696536 100644 --- a/packages/core-data/src/entities.js +++ b/packages/core-data/src/entities.js @@ -267,7 +267,7 @@ async function loadPostTypeEntities() { ] ); const postTypes = await apiFetch( { - path: '/wp/v2/types?context=view', + path: '/wp/v2/types?context=edit', } ); return Object.entries( postTypes ?? {} ).map( ( [ name, postType ] ) => { const isTemplate = [ 'wp_template', 'wp_template_part' ].includes( @@ -295,6 +295,9 @@ async function loadPostTypeEntities() { __unstablePrePersist: isTemplate ? undefined : prePersistPostType, __unstable_rest_base: postType.rest_base, syncConfig: { + enabled: Boolean( + postType.supports?.[ 'collaborative-editing' ] + ), /** * @param {Y.Doc} ydoc * @param {Object} changes @@ -338,7 +341,7 @@ async function loadPostTypeEntities() { ); }, getObjectId: ( { id } ) => id, - objectType: 'postType/' + postType.name, + objectType: `postType/${ postType.slug }`, supportsAwareness: true, supportsUndo: true, }, diff --git a/packages/core-data/src/resolvers.js b/packages/core-data/src/resolvers.js index 40d61dcd22de57..21e677294c2990 100644 --- a/packages/core-data/src/resolvers.js +++ b/packages/core-data/src/resolvers.js @@ -149,7 +149,7 @@ export const getEntityRecord = if ( window.__experimentalEnableSync && - entityConfig.syncConfig && + entityConfig.syncConfig?.enabled && ! query ) { if ( globalThis.IS_GUTENBERG_PLUGIN ) { From 81d0d133015049b6cd1969ce89a1d0c81a13e843 Mon Sep 17 00:00:00 2001 From: chriszarate Date: Thu, 21 Aug 2025 16:46:03 -0600 Subject: [PATCH 38/54] Add editor support check --- packages/core-data/src/entities.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/core-data/src/entities.js b/packages/core-data/src/entities.js index 7793728f696536..6f2d0d30b18e81 100644 --- a/packages/core-data/src/entities.js +++ b/packages/core-data/src/entities.js @@ -296,7 +296,8 @@ async function loadPostTypeEntities() { __unstable_rest_base: postType.rest_base, syncConfig: { enabled: Boolean( - postType.supports?.[ 'collaborative-editing' ] + postType.supports?.[ 'collaborative-editing' ] && + postType.supports?.editor ), /** * @param {Y.Doc} ydoc From 12cdf98db0699c42531bf49e2368aac1dee10e77 Mon Sep 17 00:00:00 2001 From: ingeniumed Date: Mon, 25 Aug 2025 14:12:41 +1000 Subject: [PATCH 39/54] Tweak the flow with comments --- packages/core-data/src/utils/crdt-blocks.ts | 28 +++++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/packages/core-data/src/utils/crdt-blocks.ts b/packages/core-data/src/utils/crdt-blocks.ts index cc10bb99bd0c5f..c270f0cb132626 100644 --- a/packages/core-data/src/utils/crdt-blocks.ts +++ b/packages/core-data/src/utils/crdt-blocks.ts @@ -8,6 +8,7 @@ import * as fun from 'lib0/function'; /** * WordPress dependencies */ +import { applyFilters } from '@wordpress/hooks'; import { RichTextData } from '@wordpress/rich-text'; import { Y } from '@wordpress/sync'; @@ -207,13 +208,24 @@ export function mergeBlocks( * @return True if the block should be synced, false otherwise. */ function shouldBlockBeSynced( block: Block ): boolean { - switch ( block.name ) { - case 'core/gallery': - return ! block.innerBlocks.some( - ( innerBlock ) => - innerBlock.attributes && innerBlock.attributes.blob - ); - default: - return true; + // Verify that the gallery block is ready to be synced. + // This means that, all images have had their blobs converted to full URLs. + // Checking for only the blobs ensures that blocks that have just been inserted work as well. + if ( 'core/gallery' === block.name ) { + return ! block.innerBlocks.some( + ( innerBlock ) => + innerBlock.attributes && innerBlock.attributes.blob + ); + } + + // Except the gallery block, all the other core blocks should be synced. + // We don't want anyone to override that, as that'll cause problems. + if ( block.name.startsWith( 'core/' ) ) { + return true; } + + // ToDo: Document this filter once its finalized. + // Allow third party blocks to customize if they are ready to be synced or not. + // It'll always be true by default as we assume it is ready to be synced. + return applyFilters( 'core.shouldBlockBeSynced', true, block ) as boolean; } From 220b6e5e04762988e302a70f95c99d38b303b077 Mon Sep 17 00:00:00 2001 From: ingeniumed Date: Tue, 26 Aug 2025 07:56:26 +1000 Subject: [PATCH 40/54] Drop the filter --- packages/core-data/src/utils/crdt-blocks.ts | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/packages/core-data/src/utils/crdt-blocks.ts b/packages/core-data/src/utils/crdt-blocks.ts index c270f0cb132626..7f7327c2782db7 100644 --- a/packages/core-data/src/utils/crdt-blocks.ts +++ b/packages/core-data/src/utils/crdt-blocks.ts @@ -8,7 +8,6 @@ import * as fun from 'lib0/function'; /** * WordPress dependencies */ -import { applyFilters } from '@wordpress/hooks'; import { RichTextData } from '@wordpress/rich-text'; import { Y } from '@wordpress/sync'; @@ -218,14 +217,6 @@ function shouldBlockBeSynced( block: Block ): boolean { ); } - // Except the gallery block, all the other core blocks should be synced. - // We don't want anyone to override that, as that'll cause problems. - if ( block.name.startsWith( 'core/' ) ) { - return true; - } - - // ToDo: Document this filter once its finalized. - // Allow third party blocks to customize if they are ready to be synced or not. - // It'll always be true by default as we assume it is ready to be synced. - return applyFilters( 'core.shouldBlockBeSynced', true, block ) as boolean; + // Allow all other blocks to be synced. + return true; } From 189697e843f9203b2a8d5db63b6be6183538902c Mon Sep 17 00:00:00 2001 From: Alec Geatches Date: Wed, 27 Aug 2025 13:31:53 -0600 Subject: [PATCH 41/54] Change 'mergeBlocks()' to 'mergeCrdtBlocks()' to make it distinct --- packages/core-data/src/utils/crdt-blocks.ts | 2 +- packages/core-data/src/utils/crdt.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/core-data/src/utils/crdt-blocks.ts b/packages/core-data/src/utils/crdt-blocks.ts index 7f7327c2782db7..fd54c0a1659903 100644 --- a/packages/core-data/src/utils/crdt-blocks.ts +++ b/packages/core-data/src/utils/crdt-blocks.ts @@ -90,7 +90,7 @@ function areBlocksEqual( gblock: Block, yblock: YBlock ): boolean { ); } -export function mergeBlocks( +export function mergeCrdtBlocks( yblocks: Y.Array< YBlock >, newValue: Block[] | Y.Array< YBlock >, _origin: string // eslint-disable-line @typescript-eslint/no-unused-vars diff --git a/packages/core-data/src/utils/crdt.ts b/packages/core-data/src/utils/crdt.ts index 5cfb047a16b310..c9e74a532e36d5 100644 --- a/packages/core-data/src/utils/crdt.ts +++ b/packages/core-data/src/utils/crdt.ts @@ -11,7 +11,7 @@ import { type CRDTDoc, Y } from '@wordpress/sync'; /** * Internal dependencies */ -import { mergeBlocks, type Block, type YBlock } from './crdt-blocks'; +import { mergeCrdtBlocks, type Block, type YBlock } from './crdt-blocks'; type PrimitiveValue = string | number | boolean | null | undefined; @@ -52,7 +52,7 @@ export function defaultApplyChangesToCRDTDoc( // Merge blocks does not need `setValue` because it has been // called above and the result can be operated on directly. - mergeBlocks( currentBlocks, newBlocks, origin ); + mergeCrdtBlocks( currentBlocks, newBlocks, origin ); break; } From be6e0bebae3fe2812b4192d02ecba551e1282b81 Mon Sep 17 00:00:00 2001 From: chriszarate Date: Wed, 27 Aug 2025 13:41:37 -0600 Subject: [PATCH 42/54] Provide bindings for CRDT persistence --- packages/core-data/src/actions.js | 15 +++++++ packages/sync/src/provider.ts | 69 ++++++++++++++++++++++++++----- packages/sync/src/types.ts | 14 +++++-- 3 files changed, 84 insertions(+), 14 deletions(-) diff --git a/packages/core-data/src/actions.js b/packages/core-data/src/actions.js index 3e729a3fbfd7ed..1aa563e64c3890 100644 --- a/packages/core-data/src/actions.js +++ b/packages/core-data/src/actions.js @@ -675,6 +675,21 @@ export const saveEntityRecord = ), }; } + if ( + window.__experimentalEnableSync && + entityConfig.syncConfig?.enabled + ) { + // Allow sync provider to create meta for the entity before persisting. + edits.meta = { + ...edits.meta, + ...( await getSyncProvider().createEntityMeta( + entityConfig.syncConfig, + persistedRecord, + edits + ) ), + }; + } + updatedRecord = await __unstableFetch( { path, method: recordId ? 'PUT' : 'POST', diff --git a/packages/sync/src/provider.ts b/packages/sync/src/provider.ts index 69297d4c5b25e1..b936fa407d014e 100644 --- a/packages/sync/src/provider.ts +++ b/packages/sync/src/provider.ts @@ -124,18 +124,14 @@ export class SyncProvider { ydoc, } ); - // Get the initial data to be synced for this record. - const initialCRDTDoc = await this.getInitialCRDTDoc( - syncConfig, - record - ); + // Get the initial document state. + const initialDoc = await this.getInitialCRDTDoc( syncConfig, record ); - // Create the initial document, possible from persisted doc. + // Apply the initial document to the current document as a singular update. Y.transact( ydoc, () => { - // apply remote changes - Y.applyUpdate( ydoc, Y.encodeStateAsUpdate( initialCRDTDoc ) ); + Y.applyUpdate( ydoc, Y.encodeStateAsUpdate( initialDoc ) ); }, 'syncProvider.bootstrap', false @@ -183,9 +179,24 @@ export class SyncProvider { record: ObjectData ): Promise< CRDTDoc > { // IMPORTANT: We use a new Yjs document so that the initial state can be - // applied to the "real" Yjs document as a singular update. + // applied to the "real" Yjs document as a singular update. Therefore, we + // don't need to wrap the changes in a transaction. const initialStateDoc = new Y.Doc( { meta: new Map() } ); + // Load the persisted document from previous sessions. + const persistedDoc = await this.getPersistedCRDTDoc( + syncConfig, + record + ); + + // If it exists, apply it as the base state of the initial document. + if ( persistedDoc ) { + Y.applyUpdate( + initialStateDoc, + Y.encodeStateAsUpdate( persistedDoc ) + ); + } + const initialData = syncConfig.getInitialObjectData( record ); syncConfig.applyChangesToCRDTDoc( initialStateDoc, @@ -196,6 +207,44 @@ export class SyncProvider { return initialStateDoc; } + /* eslint-disable @typescript-eslint/no-unused-vars */ + + /** + * Create meta for the entity, e.g., to persist the CRDT doc against the + * entity. Custom sync providers can override this method to provide their + * implementation. + * + * @param {SyncConfig} _syncConfig Sync configuration for the object type. + * @param {ObjectData} _record Record representing this object type. + * @param {Partial< ObjectData >} _changes Updates to make. + * @return {Promise< Record< string, any > >} Entity meta. + */ + public async createEntityMeta( + _syncConfig: SyncConfig, + _record: ObjectData, + _changes: Partial< ObjectData > + ): Promise< Record< string, any > > { + return Promise.resolve( {} ); + } + + /** + * Get the persisted CRDT document from the object data, e.g., from meta. + * Custom sync providers can override this method to provide their + * implementation. + * + * @param {SyncConfig} _syncConfig Sync configuration for the object type. + * @param {ObjectData} _record Record representing this object type. + * @return {Promise< CRDTDoc | null >} The persisted CRDT document, or null if none exists. + */ + protected async getPersistedCRDTDoc( + _syncConfig: SyncConfig, + _record: ObjectData + ): Promise< CRDTDoc | null > { + return Promise.resolve( null ); + } + + /* eslint-enable @typescript-eslint/no-unused-vars */ + /** * Get the undo manager. * @@ -206,7 +255,7 @@ export class SyncProvider { } /** - * Fetch data from local database or remote source. + * Update CRDT document with changes from the local store. * * @param {ObjectType} objectType Object type to load. * @param {ObjectData} record Record to load. diff --git a/packages/sync/src/types.ts b/packages/sync/src/types.ts index 7561c8afe1671a..41643a6d36d94a 100644 --- a/packages/sync/src/types.ts +++ b/packages/sync/src/types.ts @@ -5,18 +5,24 @@ import type * as Y from 'yjs'; import type { Awareness } from 'y-protocols/awareness'; export type * as Y from 'yjs'; +export type CRDTDoc = Y.Doc; export type EntityID = string; export type ObjectID = string; export type ObjectType = string; -export type ObjectData = object; export type UndoManager = Y.UndoManager; -export type CRDTDoc = Y.Doc; +// Object data represents any entity record, post, term, user, site, etc. There +// are not many expectations that can hold on its shape, but defining some +// optional properties cuts down on the type narrowing. +export interface ObjectData extends Record< string, unknown > { + meta?: Record< string, unknown >; + status?: string; +} -export type ConnectDocResult = { +export interface ConnectDocResult { awareness?: Awareness; destroy: () => void; -}; +} export type ConnectDoc = ( id: ObjectID, From ec2487294f900b7115dca12c95fd19c551021d31 Mon Sep 17 00:00:00 2001 From: Alec Geatches Date: Wed, 27 Aug 2025 15:59:44 -0600 Subject: [PATCH 43/54] Change mergeCrdtBlocks() to use direct Y types for yblocks. First step to standardizing Y types stored in the ydoc --- packages/core-data/src/utils/crdt-blocks.ts | 77 +++++++++++++++------ packages/core-data/src/utils/crdt.ts | 4 +- 2 files changed, 57 insertions(+), 24 deletions(-) diff --git a/packages/core-data/src/utils/crdt-blocks.ts b/packages/core-data/src/utils/crdt-blocks.ts index fd54c0a1659903..ab10cca89eae7c 100644 --- a/packages/core-data/src/utils/crdt-blocks.ts +++ b/packages/core-data/src/utils/crdt-blocks.ts @@ -24,11 +24,22 @@ export interface Block { name: string; } +export type YBlock = Y.Map< + /* name, clientId, and originalContent are strings. */ + | string + /* validationIssues? is an array of strings. */ + | string[] + /* attributes is a Y.Map< unknown >. */ + | Y.Map< unknown > + /* innerBlocks is a Y.Array< YBlock >. */ + | Y.Array< YBlock > +>; + // The Y.Map type is not easy to work with. The generic type it accepts represents // the possible values of the map, which are varied in our case. This type is // accurate, but will require aggressive type narrowing when the map values are // accessed -- or type casting with `as`. -export type YBlock = Y.Map< Block[ keyof Block ] >; +// export type YBlock = Y.Map< Block[ keyof Block ] >; const serializableBlocksCache = new WeakMap< WeakKey, Block[] >(); @@ -90,22 +101,31 @@ function areBlocksEqual( gblock: Block, yblock: YBlock ): boolean { ); } +/** + * Merge incoming block data into the local Y.Doc. + * This function is called to sync local block changes to a shared Y.Doc. + * + * @param yblocks The blocks in the local Y.Doc. + * @param incomingBlocks Gutenberg blocks being synced. + * @param _origin The origin of the sync, either 'syncProvider.getInitialCRDTDoc' or 'gutenberg'. + */ + export function mergeCrdtBlocks( - yblocks: Y.Array< YBlock >, - newValue: Block[] | Y.Array< YBlock >, + yblocks: Y.Array< YBlock >, // yblocks represent the blocks in the local Y.Doc + incomingBlocks: Block[], // incomingBlocks represent JSON blocks being synced, either from a peer or from the local editor _origin: string // eslint-disable-line @typescript-eslint/no-unused-vars ): void { // Ensure we are working with serializable block data. - if ( ! serializableBlocksCache.has( newValue ) ) { + if ( ! serializableBlocksCache.has( incomingBlocks ) ) { serializableBlocksCache.set( - newValue, - makeBlocksSerializable( newValue ) + incomingBlocks, + makeBlocksSerializable( incomingBlocks ) ); } - const unfilteredBlocks = serializableBlocksCache.get( newValue ) ?? []; + const allBlocks = serializableBlocksCache.get( incomingBlocks ) ?? []; // Ensure we skip blocks that we don't want to sync at the moment - const blocks = unfilteredBlocks.filter( ( block ) => + const blocksToSync = allBlocks.filter( ( block ) => shouldBlockBeSynced( block ) ); @@ -119,8 +139,10 @@ export function mergeCrdtBlocks( // E.g.: // - textual content (using rich-text formatting?) may always be stored under `block.text` // - local information that shouldn't be shared (e.g. clientId or isDragging) is stored under `block.private` - - const numOfCommonEntries = math.min( blocks.length ?? 0, yblocks.length ); + const numOfCommonEntries = math.min( + blocksToSync.length ?? 0, + yblocks.length + ); let left = 0; let right = 0; @@ -129,7 +151,7 @@ export function mergeCrdtBlocks( for ( ; left < numOfCommonEntries && - areBlocksEqual( blocks[ left ], yblocks.get( left ) ); + areBlocksEqual( blocksToSync[ left ], yblocks.get( left ) ); left++ ) { /* nop */ @@ -140,7 +162,7 @@ export function mergeCrdtBlocks( ; right < numOfCommonEntries - left && areBlocksEqual( - blocks[ blocks.length - right - 1 ], + blocksToSync[ blocksToSync.length - right - 1 ], yblocks.get( yblocks.length - right - 1 ) ); right++ @@ -149,15 +171,28 @@ export function mergeCrdtBlocks( } const numOfUpdatesNeeded = numOfCommonEntries - left - right; - const numOfInsertionsNeeded = math.max( 0, blocks.length - yblocks.length ); - const numOfDeletionsNeeded = math.max( 0, yblocks.length - blocks.length ); + const numOfInsertionsNeeded = math.max( + 0, + blocksToSync.length - yblocks.length + ); + const numOfDeletionsNeeded = math.max( + 0, + yblocks.length - blocksToSync.length + ); // updates for ( let i = 0; i < numOfUpdatesNeeded; i++, left++ ) { - const block = blocks[ left ]; + const block = blocksToSync[ left ]; const yblock = yblocks.get( left ); Object.entries( block ).forEach( ( [ k, v ] ) => { if ( ! fun.equalityDeep( block[ k ], yblock.get( k ) ) ) { + if ( k === 'innerBlocks' ) { + // Recursively merge innerBlocks + const yInnerBlocks = yblock.get( k ) as Y.Array< YBlock >; + + mergeCrdtBlocks( yInnerBlocks, v, _origin ); + } + yblock.set( k, v ); } } ); @@ -173,17 +208,17 @@ export function mergeCrdtBlocks( // inserts for ( let i = 0; i < numOfInsertionsNeeded; i++, left++ ) { - yblocks.insert( left, [ - new Y.Map< Block[ keyof Block ] >( - Object.entries( blocks[ left ] ) - ), - ] ); + const newBlock = [ + new Y.Map( Object.entries( blocksToSync[ left ] ) ) as YBlock, + ]; + + yblocks.insert( left, newBlock ); } // remove duplicate clientids const knownClientIds = new Set< string >(); for ( let j = 0; j < yblocks.length; j++ ) { - const yblock: Y.Map< Block[ keyof Block ] > = yblocks.get( j ); + const yblock: YBlock = yblocks.get( j ); let clientId: string = yblock.get( 'clientId' ) as string; diff --git a/packages/core-data/src/utils/crdt.ts b/packages/core-data/src/utils/crdt.ts index c9e74a532e36d5..3ccbd67cafd0ab 100644 --- a/packages/core-data/src/utils/crdt.ts +++ b/packages/core-data/src/utils/crdt.ts @@ -37,9 +37,7 @@ export function defaultApplyChangesToCRDTDoc( switch ( key ) { case 'blocks': { - let currentBlocks = ymap.get( - 'blocks' - ) as PostChanges[ 'blocks' ]; + let currentBlocks = ymap.get( 'blocks' ) as Y.Array< YBlock >; if ( ! ( currentBlocks instanceof Y.Array ) ) { currentBlocks = setValue< Y.Array< YBlock > >( From a41e5181265b3ef26a745a9c442bf9eb32544308 Mon Sep 17 00:00:00 2001 From: chriszarate Date: Thu, 28 Aug 2025 10:29:16 -0600 Subject: [PATCH 44/54] Control and validate CRDT doc version internally --- packages/sync/src/index.ts | 2 +- packages/sync/src/provider.ts | 33 ++++++++++++++++++++++++--------- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/packages/sync/src/index.ts b/packages/sync/src/index.ts index a630969d6d49b8..ec0341e9dc44f0 100644 --- a/packages/sync/src/index.ts +++ b/packages/sync/src/index.ts @@ -12,7 +12,7 @@ import { SyncProvider } from './provider'; export * as Y from 'yjs'; export { connectIndexDb } from './connect-indexdb'; export { createWebRTCConnection } from './create-webrtc-connection'; -export { SyncProvider } from './provider'; +export { CRDT_DOC_VERSION, SyncProvider } from './provider'; export * from './types'; declare global { diff --git a/packages/sync/src/provider.ts b/packages/sync/src/provider.ts index b936fa407d014e..35e930b8fc5e09 100644 --- a/packages/sync/src/provider.ts +++ b/packages/sync/src/provider.ts @@ -23,6 +23,11 @@ interface EntityState { ydoc: CRDTDoc; } +// This version number should be incremented whenever there are breaking changes +// to Yjs doc schema or in how it is interpreted by code in the SyncConfig. This +// allows implementors to invalidate persisted CRDT docs, if any. +export const CRDT_DOC_VERSION = 1; + export class SyncProvider { private connectLocal: ConnectDoc | null; private connectRemote: ConnectDoc | null; @@ -91,7 +96,10 @@ export class SyncProvider { record: ObjectData, handleChanges: ( data: Partial< ObjectData > ) => void ): Promise< void > { - const ydoc = new Y.Doc( { meta: new Map() } ); + const meta = new Map< string, unknown >( [ + [ 'version', CRDT_DOC_VERSION ], + ] ); + const ydoc = new Y.Doc( { meta } ); const objectId = syncConfig.getObjectId( record ); const objectType = syncConfig.objectType; const connections = await this.connect( objectId, objectType, ydoc ); @@ -174,23 +182,28 @@ export class SyncProvider { * @param {SyncConfig} syncConfig Sync configuration for the object type. * @param {ObjectData} record Initial data to apply to the document. */ - protected async getInitialCRDTDoc( + private async getInitialCRDTDoc( syncConfig: SyncConfig, record: ObjectData ): Promise< CRDTDoc > { // IMPORTANT: We use a new Yjs document so that the initial state can be // applied to the "real" Yjs document as a singular update. Therefore, we // don't need to wrap the changes in a transaction. - const initialStateDoc = new Y.Doc( { meta: new Map() } ); + const initialStateDoc = new Y.Doc(); // Load the persisted document from previous sessions. const persistedDoc = await this.getPersistedCRDTDoc( syncConfig, - record + record, + CRDT_DOC_VERSION ); - // If it exists, apply it as the base state of the initial document. - if ( persistedDoc ) { + // If it exists and matches the current version, apply it as the base state + // of the initial document. + if ( + persistedDoc && + CRDT_DOC_VERSION === persistedDoc.meta?.get( 'version' ) + ) { Y.applyUpdate( initialStateDoc, Y.encodeStateAsUpdate( persistedDoc ) @@ -232,13 +245,15 @@ export class SyncProvider { * Custom sync providers can override this method to provide their * implementation. * - * @param {SyncConfig} _syncConfig Sync configuration for the object type. - * @param {ObjectData} _record Record representing this object type. + * @param {SyncConfig} _syncConfig Sync configuration for the object type. + * @param {ObjectData} _record Record representing this object type. + * @param {number} _expectedVersion Expected version of persisted CRDT document. * @return {Promise< CRDTDoc | null >} The persisted CRDT document, or null if none exists. */ protected async getPersistedCRDTDoc( _syncConfig: SyncConfig, - _record: ObjectData + _record: ObjectData, + _expectedVersion: number ): Promise< CRDTDoc | null > { return Promise.resolve( null ); } From ec8df1ba05093c0308acce434cfcb6ee7094953f Mon Sep 17 00:00:00 2001 From: Alec Geatches Date: Thu, 28 Aug 2025 11:03:58 -0600 Subject: [PATCH 45/54] Fix CRDT merge object equality check against yBlockAsJson --- packages/core-data/src/utils/crdt-blocks.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core-data/src/utils/crdt-blocks.ts b/packages/core-data/src/utils/crdt-blocks.ts index ab10cca89eae7c..033e6c6432394f 100644 --- a/packages/core-data/src/utils/crdt-blocks.ts +++ b/packages/core-data/src/utils/crdt-blocks.ts @@ -88,7 +88,7 @@ function areBlocksEqual( gblock: Block, yblock: YBlock ): boolean { }; const res = fun.equalityDeep( Object.assign( {}, gblock, overwrites ), - Object.assign( {}, yblock, overwrites ) + Object.assign( {}, yblockAsJson, overwrites ) ); const inners = gblock.innerBlocks || []; const yinners = yblockAsJson.innerBlocks || []; From dd71bdf923458ee85234b1bdd3d3051519ee61ba Mon Sep 17 00:00:00 2001 From: chriszarate Date: Thu, 28 Aug 2025 11:54:58 -0600 Subject: [PATCH 46/54] Add createNewYBlock for recursive insert --- packages/core-data/src/utils/crdt-blocks.ts | 32 +++++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/packages/core-data/src/utils/crdt-blocks.ts b/packages/core-data/src/utils/crdt-blocks.ts index 033e6c6432394f..40176ef06e3828 100644 --- a/packages/core-data/src/utils/crdt-blocks.ts +++ b/packages/core-data/src/utils/crdt-blocks.ts @@ -101,6 +101,34 @@ function areBlocksEqual( gblock: Block, yblock: YBlock ): boolean { ); } +function createNewYBlock( block: Block ): YBlock { + return new Y.Map( + Object.entries( block ).map( ( [ key, value ] ) => { + switch ( key ) { + case 'innerBlocks': { + if ( Array.isArray( value ) ) { + const innerBlocks = new Y.Array(); + + innerBlocks.insert( + 0, + value.map( ( innerBlock: Block ) => + createNewYBlock( innerBlock ) + ) + ); + + return [ key, innerBlocks ]; + } + + return [ key, value ]; + } + + default: + return [ key, value ]; + } + } ) + ); +} + /** * Merge incoming block data into the local Y.Doc. * This function is called to sync local block changes to a shared Y.Doc. @@ -208,9 +236,7 @@ export function mergeCrdtBlocks( // inserts for ( let i = 0; i < numOfInsertionsNeeded; i++, left++ ) { - const newBlock = [ - new Y.Map( Object.entries( blocksToSync[ left ] ) ) as YBlock, - ]; + const newBlock = [ createNewYBlock( blocksToSync[ left ] ) ]; yblocks.insert( left, newBlock ); } From c12aa9be5f6e7673670290e59764527669befcf8 Mon Sep 17 00:00:00 2001 From: chriszarate Date: Thu, 28 Aug 2025 11:58:43 -0600 Subject: [PATCH 47/54] Ensure we always recursively merge innerBlocks --- packages/core-data/src/utils/crdt-blocks.ts | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/packages/core-data/src/utils/crdt-blocks.ts b/packages/core-data/src/utils/crdt-blocks.ts index 40176ef06e3828..3bdeeee5b6edcf 100644 --- a/packages/core-data/src/utils/crdt-blocks.ts +++ b/packages/core-data/src/utils/crdt-blocks.ts @@ -137,7 +137,6 @@ function createNewYBlock( block: Block ): YBlock { * @param incomingBlocks Gutenberg blocks being synced. * @param _origin The origin of the sync, either 'syncProvider.getInitialCRDTDoc' or 'gutenberg'. */ - export function mergeCrdtBlocks( yblocks: Y.Array< YBlock >, // yblocks represent the blocks in the local Y.Doc incomingBlocks: Block[], // incomingBlocks represent JSON blocks being synced, either from a peer or from the local editor @@ -212,16 +211,21 @@ export function mergeCrdtBlocks( for ( let i = 0; i < numOfUpdatesNeeded; i++, left++ ) { const block = blocksToSync[ left ]; const yblock = yblocks.get( left ); - Object.entries( block ).forEach( ( [ k, v ] ) => { - if ( ! fun.equalityDeep( block[ k ], yblock.get( k ) ) ) { - if ( k === 'innerBlocks' ) { + Object.entries( block ).forEach( ( [ key, value ] ) => { + switch ( key ) { + case 'innerBlocks': { // Recursively merge innerBlocks - const yInnerBlocks = yblock.get( k ) as Y.Array< YBlock >; - - mergeCrdtBlocks( yInnerBlocks, v, _origin ); + const yInnerBlocks = yblock.get( key ) as Y.Array< YBlock >; + mergeCrdtBlocks( yInnerBlocks, value ?? [], _origin ); + break; } - yblock.set( k, v ); + default: + if ( + ! fun.equalityDeep( block[ key ], yblock.get( key ) ) + ) { + yblock.set( key, value ); + } } } ); yblock.forEach( ( _v, k ) => { From 131f0e5f0f858d172cb8e8178fdfbc5ab8628372 Mon Sep 17 00:00:00 2001 From: Alec Geatches Date: Thu, 28 Aug 2025 12:14:57 -0600 Subject: [PATCH 48/54] In areBlocksEqual(), ensure YBlock type instead of JSON value for comparison --- packages/core-data/src/utils/crdt-blocks.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/core-data/src/utils/crdt-blocks.ts b/packages/core-data/src/utils/crdt-blocks.ts index 3bdeeee5b6edcf..88f1ef87ffed43 100644 --- a/packages/core-data/src/utils/crdt-blocks.ts +++ b/packages/core-data/src/utils/crdt-blocks.ts @@ -91,12 +91,12 @@ function areBlocksEqual( gblock: Block, yblock: YBlock ): boolean { Object.assign( {}, yblockAsJson, overwrites ) ); const inners = gblock.innerBlocks || []; - const yinners = yblockAsJson.innerBlocks || []; + const yinners = yblock.get( 'innerBlocks' ) as Y.Array< YBlock >; return ( res && inners.length === yinners.length && inners.every( ( block: Block, i: number ) => - areBlocksEqual( block, yinners[ i ] ) + areBlocksEqual( block, yinners.get( i ) ) ) ); } From a3720b667b62aebded997c7fd6914fd90404bda8 Mon Sep 17 00:00:00 2001 From: Alec Geatches Date: Thu, 28 Aug 2025 12:25:41 -0600 Subject: [PATCH 49/54] Use Y.Map type for attributes in yblocks --- packages/core-data/src/utils/crdt-blocks.ts | 24 +++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/packages/core-data/src/utils/crdt-blocks.ts b/packages/core-data/src/utils/crdt-blocks.ts index 88f1ef87ffed43..da2897a24c0d9e 100644 --- a/packages/core-data/src/utils/crdt-blocks.ts +++ b/packages/core-data/src/utils/crdt-blocks.ts @@ -122,6 +122,19 @@ function createNewYBlock( block: Block ): YBlock { return [ key, value ]; } + case 'attributes': { + const attributes = new Y.Map( + Object.entries( value ).map( + ( [ attributeKey, attributeValue ] ) => { + // Rich-text logic here + return [ attributeKey, attributeValue ]; + } + ) + ); + + return [ key, attributes ]; + } + default: return [ key, value ]; } @@ -220,6 +233,17 @@ export function mergeCrdtBlocks( break; } + case 'attributes': { + const yAttributes = yblock.get( key ) as Y.Map< unknown >; + Object.entries( value ).forEach( + ( [ attributeKey, attributeValue ] ) => { + // Rich-text logic here + yAttributes.set( attributeKey, attributeValue ); + } + ); + break; + } + default: if ( ! fun.equalityDeep( block[ key ], yblock.get( key ) ) From 3b116040a43367448206682ae399ac8578ecf853 Mon Sep 17 00:00:00 2001 From: chriszarate Date: Thu, 28 Aug 2025 12:32:26 -0600 Subject: [PATCH 50/54] Bugfix: Don't allow non-array values for innerBlocks --- packages/core-data/src/utils/crdt-blocks.ts | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/packages/core-data/src/utils/crdt-blocks.ts b/packages/core-data/src/utils/crdt-blocks.ts index da2897a24c0d9e..5c6841d24d9dac 100644 --- a/packages/core-data/src/utils/crdt-blocks.ts +++ b/packages/core-data/src/utils/crdt-blocks.ts @@ -106,20 +106,21 @@ function createNewYBlock( block: Block ): YBlock { Object.entries( block ).map( ( [ key, value ] ) => { switch ( key ) { case 'innerBlocks': { - if ( Array.isArray( value ) ) { - const innerBlocks = new Y.Array(); - - innerBlocks.insert( - 0, - value.map( ( innerBlock: Block ) => - createNewYBlock( innerBlock ) - ) - ); + const innerBlocks = new Y.Array(); + // If not an array, set to empty Y.Array. + if ( ! Array.isArray( value ) ) { return [ key, innerBlocks ]; } - return [ key, value ]; + innerBlocks.insert( + 0, + value.map( ( innerBlock: Block ) => + createNewYBlock( innerBlock ) + ) + ); + + return [ key, innerBlocks ]; } case 'attributes': { From cf9a1cef8f067c13746bf786f2233cfe54eef5b7 Mon Sep 17 00:00:00 2001 From: Alec Geatches Date: Thu, 28 Aug 2025 12:53:39 -0600 Subject: [PATCH 51/54] Cast rich-text types into Y.Text in the ydoc --- packages/core-data/src/utils/crdt-blocks.ts | 76 ++++++++++++++++++++- 1 file changed, 73 insertions(+), 3 deletions(-) diff --git a/packages/core-data/src/utils/crdt-blocks.ts b/packages/core-data/src/utils/crdt-blocks.ts index 5c6841d24d9dac..1b40ca07c9ed50 100644 --- a/packages/core-data/src/utils/crdt-blocks.ts +++ b/packages/core-data/src/utils/crdt-blocks.ts @@ -11,6 +11,9 @@ import * as fun from 'lib0/function'; import { RichTextData } from '@wordpress/rich-text'; import { Y } from '@wordpress/sync'; +// @ts-expect-error - This is a TypeScript file, and @wordpress/blocks doesn't have a tsconfig.json? +import { getBlockTypes } from '@wordpress/blocks'; + interface BlockAttributes { [ key: string ]: unknown; } @@ -127,7 +130,18 @@ function createNewYBlock( block: Block ): YBlock { const attributes = new Y.Map( Object.entries( value ).map( ( [ attributeKey, attributeValue ] ) => { - // Rich-text logic here + const isRichText = isRichTextAttribute( + block.name, + attributeKey + ); + + if ( isRichText ) { + return [ + attributeKey, + new Y.Text( attributeValue as string ), + ]; + } + return [ attributeKey, attributeValue ]; } ) @@ -238,8 +252,19 @@ export function mergeCrdtBlocks( const yAttributes = yblock.get( key ) as Y.Map< unknown >; Object.entries( value ).forEach( ( [ attributeKey, attributeValue ] ) => { - // Rich-text logic here - yAttributes.set( attributeKey, attributeValue ); + const isRichText = isRichTextAttribute( + block.name, + attributeKey + ); + + if ( isRichText ) { + const ytext = new Y.Text( + attributeValue as string + ); + yAttributes.set( attributeKey, ytext ); + } else { + yAttributes.set( attributeKey, attributeValue ); + } } ); break; @@ -310,3 +335,48 @@ function shouldBlockBeSynced( block: Block ): boolean { // Allow all other blocks to be synced. return true; } + +// Cache rich-text attributes for looked-up block types. +const cachedRichTextAttributes = new Map< string, Map< string, true > >(); + +/** + * Given a block name and attribute key, return true if the attribute is rich-text typed. + * + * @param blockName The name of the block, e.g. 'core/paragraph'. + * @param attributeKey The key of the attribute to check, e.g. 'content'. + * @return True if the attribute is rich-text typed, false otherwise. + */ +function isRichTextAttribute( + blockName: string, + attributeKey: string +): boolean { + if ( cachedRichTextAttributes.has( blockName ) ) { + // If we've already cached the rich-text attributes for this block type, + // return the cached value. + return ( + cachedRichTextAttributes.get( blockName )?.has( attributeKey ) ?? + false + ); + } + + const allRegisteredBlockTypes = getBlockTypes(); + const matchingBlockType = allRegisteredBlockTypes.find( + ( blockType ) => blockType.name === blockName + ); + + const isBlockTypeRegistered = matchingBlockType !== undefined; + const richTextAttributeMap = new Map< string, true >(); + + if ( isBlockTypeRegistered ) { + for ( const [ registeredKey, registeredProperties ] of Object.entries( + matchingBlockType.attributes as Record< string, { type: string } > + ) ) { + if ( registeredProperties.type === 'rich-text' ) { + richTextAttributeMap.set( registeredKey, true ); + } + } + } + + cachedRichTextAttributes.set( blockName, richTextAttributeMap ); + return richTextAttributeMap.has( attributeKey ); +} From 9a67df3402ca7ea2ee98a44813af853331f659fb Mon Sep 17 00:00:00 2001 From: chriszarate Date: Thu, 28 Aug 2025 14:58:04 -0600 Subject: [PATCH 52/54] Improve type safety, DRY up, and slightly more efficient --- packages/core-data/src/utils/crdt-blocks.ts | 148 ++++++++++---------- 1 file changed, 72 insertions(+), 76 deletions(-) diff --git a/packages/core-data/src/utils/crdt-blocks.ts b/packages/core-data/src/utils/crdt-blocks.ts index 1b40ca07c9ed50..f4fde8f64e074e 100644 --- a/packages/core-data/src/utils/crdt-blocks.ts +++ b/packages/core-data/src/utils/crdt-blocks.ts @@ -18,6 +18,11 @@ interface BlockAttributes { [ key: string ]: unknown; } +interface BlockType { + name: string; + attributes?: Record< string, { type?: string } >; +} + export interface Block { attributes: BlockAttributes; clientId?: string; @@ -104,10 +109,39 @@ function areBlocksEqual( gblock: Block, yblock: YBlock ): boolean { ); } +function createNewYAttributeMap( + blockName: string, + attributes: BlockAttributes +): Y.Map< Y.Text | unknown > { + return new Y.Map( + Object.entries( attributes ).map( + ( [ attributeKey, attributeValue ] ) => { + const isRichText = isRichTextAttribute( + blockName, + attributeKey + ); + + if ( isRichText && 'string' === typeof attributeValue ) { + return [ + attributeKey, + new Y.Text( attributeValue as string ), + ]; + } + + return [ attributeKey, attributeValue ]; + } + ) + ); +} + function createNewYBlock( block: Block ): YBlock { return new Y.Map( Object.entries( block ).map( ( [ key, value ] ) => { switch ( key ) { + case 'attributes': { + return [ key, createNewYAttributeMap( block.name, value ) ]; + } + case 'innerBlocks': { const innerBlocks = new Y.Array(); @@ -126,30 +160,6 @@ function createNewYBlock( block: Block ): YBlock { return [ key, innerBlocks ]; } - case 'attributes': { - const attributes = new Y.Map( - Object.entries( value ).map( - ( [ attributeKey, attributeValue ] ) => { - const isRichText = isRichTextAttribute( - block.name, - attributeKey - ); - - if ( isRichText ) { - return [ - attributeKey, - new Y.Text( attributeValue as string ), - ]; - } - - return [ attributeKey, attributeValue ]; - } - ) - ); - - return [ key, attributes ]; - } - default: return [ key, value ]; } @@ -241,6 +251,18 @@ export function mergeCrdtBlocks( const yblock = yblocks.get( left ); Object.entries( block ).forEach( ( [ key, value ] ) => { switch ( key ) { + case 'attributes': { + if ( + ! fun.equalityDeep( block[ key ], yblock.get( key ) ) + ) { + yblock.set( + key, + createNewYAttributeMap( block.name, value ) + ); + } + break; + } + case 'innerBlocks': { // Recursively merge innerBlocks const yInnerBlocks = yblock.get( key ) as Y.Array< YBlock >; @@ -248,28 +270,6 @@ export function mergeCrdtBlocks( break; } - case 'attributes': { - const yAttributes = yblock.get( key ) as Y.Map< unknown >; - Object.entries( value ).forEach( - ( [ attributeKey, attributeValue ] ) => { - const isRichText = isRichTextAttribute( - block.name, - attributeKey - ); - - if ( isRichText ) { - const ytext = new Y.Text( - attributeValue as string - ); - yAttributes.set( attributeKey, ytext ); - } else { - yAttributes.set( attributeKey, attributeValue ); - } - } - ); - break; - } - default: if ( ! fun.equalityDeep( block[ key ], yblock.get( key ) ) @@ -336,47 +336,43 @@ function shouldBlockBeSynced( block: Block ): boolean { return true; } -// Cache rich-text attributes for looked-up block types. -const cachedRichTextAttributes = new Map< string, Map< string, true > >(); +// Cache rich-text attributes for all block types. +let cachedRichTextAttributes: Map< string, Map< string, true > >; /** * Given a block name and attribute key, return true if the attribute is rich-text typed. * - * @param blockName The name of the block, e.g. 'core/paragraph'. - * @param attributeKey The key of the attribute to check, e.g. 'content'. + * @param blockName The name of the block, e.g. 'core/paragraph'. + * @param attributeName The name of the attribute to check, e.g. 'content'. * @return True if the attribute is rich-text typed, false otherwise. */ function isRichTextAttribute( blockName: string, - attributeKey: string + attributeName: string ): boolean { - if ( cachedRichTextAttributes.has( blockName ) ) { - // If we've already cached the rich-text attributes for this block type, - // return the cached value. - return ( - cachedRichTextAttributes.get( blockName )?.has( attributeKey ) ?? - false - ); - } - - const allRegisteredBlockTypes = getBlockTypes(); - const matchingBlockType = allRegisteredBlockTypes.find( - ( blockType ) => blockType.name === blockName - ); - - const isBlockTypeRegistered = matchingBlockType !== undefined; - const richTextAttributeMap = new Map< string, true >(); - - if ( isBlockTypeRegistered ) { - for ( const [ registeredKey, registeredProperties ] of Object.entries( - matchingBlockType.attributes as Record< string, { type: string } > - ) ) { - if ( registeredProperties.type === 'rich-text' ) { - richTextAttributeMap.set( registeredKey, true ); + if ( ! cachedRichTextAttributes ) { + // Parse the attributes for all blocks once. + cachedRichTextAttributes = new Map< string, Map< string, true > >(); + + for ( const blockType of getBlockTypes() as BlockType[] ) { + const richTextAttributeMap = new Map< string, true >(); + + for ( const [ name, definition ] of Object.entries( + blockType.attributes ?? {} + ) ) { + if ( 'rich-text' === definition.type ) { + richTextAttributeMap.set( name, true ); + } } + + cachedRichTextAttributes.set( + blockType.name, + richTextAttributeMap + ); } } - cachedRichTextAttributes.set( blockName, richTextAttributeMap ); - return richTextAttributeMap.has( attributeKey ); + return ( + cachedRichTextAttributes.get( blockName )?.has( attributeName ) ?? false + ); } From a7e82289d5dc99f90bd6b93ac5440cfed4f6dd55 Mon Sep 17 00:00:00 2001 From: chriszarate Date: Thu, 28 Aug 2025 15:30:16 -0600 Subject: [PATCH 53/54] Ensure attributes are deleted when removed --- packages/core-data/src/utils/crdt-blocks.ts | 86 +++++++++++++++------ 1 file changed, 62 insertions(+), 24 deletions(-) diff --git a/packages/core-data/src/utils/crdt-blocks.ts b/packages/core-data/src/utils/crdt-blocks.ts index f4fde8f64e074e..00fc83ba0f8819 100644 --- a/packages/core-data/src/utils/crdt-blocks.ts +++ b/packages/core-data/src/utils/crdt-blocks.ts @@ -38,11 +38,13 @@ export type YBlock = Y.Map< /* validationIssues? is an array of strings. */ | string[] /* attributes is a Y.Map< unknown >. */ - | Y.Map< unknown > + | YBlockAttributes /* innerBlocks is a Y.Array< YBlock >. */ | Y.Array< YBlock > >; +export type YBlockAttributes = Y.Map< Y.Text | unknown >; + // The Y.Map type is not easy to work with. The generic type it accepts represents // the possible values of the map, which are varied in our case. This type is // accurate, but will require aggressive type narrowing when the map values are @@ -112,28 +114,37 @@ function areBlocksEqual( gblock: Block, yblock: YBlock ): boolean { function createNewYAttributeMap( blockName: string, attributes: BlockAttributes -): Y.Map< Y.Text | unknown > { +): YBlockAttributes { return new Y.Map( Object.entries( attributes ).map( - ( [ attributeKey, attributeValue ] ) => { - const isRichText = isRichTextAttribute( - blockName, - attributeKey - ); - - if ( isRichText && 'string' === typeof attributeValue ) { - return [ - attributeKey, - new Y.Text( attributeValue as string ), - ]; - } - - return [ attributeKey, attributeValue ]; + ( [ attributeName, attributeValue ] ) => { + return [ + attributeName, + createNewYAttributeValue( + blockName, + attributeName, + attributeValue + ), + ]; } ) ); } +function createNewYAttributeValue( + blockName: string, + attributeName: string, + attributeValue: unknown +): Y.Text | unknown { + const isRichText = isRichTextAttribute( blockName, attributeName ); + + if ( isRichText && 'string' === typeof attributeValue ) { + return new Y.Text( attributeValue ); + } + + return attributeValue; +} + function createNewYBlock( block: Block ): YBlock { return new Y.Map( Object.entries( block ).map( ( [ key, value ] ) => { @@ -252,14 +263,41 @@ export function mergeCrdtBlocks( Object.entries( block ).forEach( ( [ key, value ] ) => { switch ( key ) { case 'attributes': { - if ( - ! fun.equalityDeep( block[ key ], yblock.get( key ) ) - ) { - yblock.set( - key, - createNewYAttributeMap( block.name, value ) - ); - } + const currentAttributes = + ( yblock.get( key ) as YBlockAttributes ) ?? + createNewYAttributeMap( block.name, {} ); + + Object.entries( value ).forEach( + ( [ attributeName, attributeValue ] ) => { + if ( + fun.equalityDeep( + currentAttributes.get( attributeName ), + attributeValue + ) + ) { + return; + } + + currentAttributes.set( + attributeName, + createNewYAttributeValue( + block.name, + attributeName, + attributeValue + ) + ); + } + ); + + // Delete any attributes that are no longer present. + currentAttributes.forEach( + ( _attrValue: unknown, attrName: string ) => { + if ( ! value.hasOwnProperty( attrName ) ) { + currentAttributes.delete( attrName ); + } + } + ); + break; } From 15ad2015b5ea0d573bb6786245d28deda99066af Mon Sep 17 00:00:00 2001 From: chriszarate Date: Thu, 28 Aug 2025 15:58:40 -0600 Subject: [PATCH 54/54] Bugfix: Make sure new attributes are set. --- packages/core-data/src/utils/crdt-blocks.ts | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/packages/core-data/src/utils/crdt-blocks.ts b/packages/core-data/src/utils/crdt-blocks.ts index 00fc83ba0f8819..aaf8cb6ddf4711 100644 --- a/packages/core-data/src/utils/crdt-blocks.ts +++ b/packages/core-data/src/utils/crdt-blocks.ts @@ -263,15 +263,24 @@ export function mergeCrdtBlocks( Object.entries( block ).forEach( ( [ key, value ] ) => { switch ( key ) { case 'attributes': { - const currentAttributes = - ( yblock.get( key ) as YBlockAttributes ) ?? - createNewYAttributeMap( block.name, {} ); + const currentAttributes = yblock.get( + key + ) as YBlockAttributes; + + // If attributes are not set on the yblock, use the new values. + if ( ! currentAttributes ) { + yblock.set( + key, + createNewYAttributeMap( block.name, value ) + ); + break; + } Object.entries( value ).forEach( ( [ attributeName, attributeValue ] ) => { if ( fun.equalityDeep( - currentAttributes.get( attributeName ), + currentAttributes?.get( attributeName ), attributeValue ) ) {