enh: store awareness messages separately

appinfo/info.xml

@@ -11,7 +11,7 @@
 - **💾 Open format:** Files are saved as [Markdown](https://en.wikipedia.org/wiki/Markdown), so you can edit them from any other text app too.
 - **✊ Strong foundation:** We use [🐈 tiptap](https://tiptap.scrumpy.io) which is based on [🦉 ProseMirror](https://prosemirror.net) – huge thanks to them!
 	]]></description>
-	<version>3.7.0</version>
+	<version>3.7.1</version>
 	<licence>agpl</licence>
 	<author mail="[email protected]">Julius Härtl</author>
 	<namespace>Text</namespace>

composer/composer/autoload_classmap.php

@@ -44,6 +44,7 @@
     'OCA\\Text\\Migration\\Version030201Date20201116110353' => $baseDir . '/../lib/Migration/Version030201Date20201116110353.php',
     'OCA\\Text\\Migration\\Version030201Date20201116123153' => $baseDir . '/../lib/Migration/Version030201Date20201116123153.php',
     'OCA\\Text\\Migration\\Version030501Date20220202101853' => $baseDir . '/../lib/Migration/Version030501Date20220202101853.php',
+    'OCA\\Text\\Migration\\Version030701Date20230207131313' => $baseDir . '/../lib/Migration/Version030701Date20230207131313.php',
     'OCA\\Text\\Notification\\Notifier' => $baseDir . '/../lib/Notification/Notifier.php',
     'OCA\\Text\\Service\\ApiService' => $baseDir . '/../lib/Service/ApiService.php',
     'OCA\\Text\\Service\\AttachmentService' => $baseDir . '/../lib/Service/AttachmentService.php',

composer/composer/autoload_static.php

@@ -59,6 +59,7 @@ class ComposerStaticInitText
         'OCA\\Text\\Migration\\Version030201Date20201116110353' => __DIR__ . '/..' . '/../lib/Migration/Version030201Date20201116110353.php',
         'OCA\\Text\\Migration\\Version030201Date20201116123153' => __DIR__ . '/..' . '/../lib/Migration/Version030201Date20201116123153.php',
         'OCA\\Text\\Migration\\Version030501Date20220202101853' => __DIR__ . '/..' . '/../lib/Migration/Version030501Date20220202101853.php',
+        'OCA\\Text\\Migration\\Version030701Date20230207131313' => __DIR__ . '/..' . '/../lib/Migration/Version030701Date20230207131313.php',
         'OCA\\Text\\Notification\\Notifier' => __DIR__ . '/..' . '/../lib/Notification/Notifier.php',
         'OCA\\Text\\Service\\ApiService' => __DIR__ . '/..' . '/../lib/Service/ApiService.php',
         'OCA\\Text\\Service\\AttachmentService' => __DIR__ . '/..' . '/../lib/Service/AttachmentService.php',

cypress/e2e/sync.spec.js

@@ -53,7 +53,7 @@ describe('yjs document state', () => {
 			.should('include', 'saves the doc state')
 	})
 
-	it('passes the doc state from one session to the next', () => {
+	it('passes the doc content from one session to the next', () => {
 		cy.getContent().type('{ctrl+s}')
 		cy.wait('@sync').its('request.body')
 			.should('have.property', 'documentState')
@@ -64,7 +64,7 @@ describe('yjs document state', () => {
 		cy.openTestFile()
 		cy.wait('@create', { timeout: 10000 })
 			.its('response.body')
-			.should('have.property', 'documentState')
+			.should('have.property', 'content')
 			.should('not.be.empty')
 		cy.wait('@sync').its('request.body').should('have.property', 'version')
 		cy.getContent().find('h2').should('contain', 'Hello world')

cypress/support/sessions.js

@@ -42,8 +42,8 @@ Cypress.Commands.add('failToCreateTextSession', (fileId) => {
 		.catch((err) => err.response)
 })
 
-Cypress.Commands.add('pushSteps', ({ connection, steps, version }) => {
-	return connection.push({ steps, version })
+Cypress.Commands.add('pushSteps', ({ connection, steps, version, awareness = '' }) => {
+	return connection.push({ steps, version, awareness })
 		.then(response => response.data)
 })
 

lib/Controller/PublicSessionController.php

@@ -82,8 +82,8 @@ public function close(int $documentId, int $sessionId, string $sessionToken): Da
 	 * @NoAdminRequired
 	 * @PublicPage
 	 */
-	public function push(int $documentId, int $sessionId, string $sessionToken, int $version, array $steps, string $token): DataResponse {
-		return $this->apiService->push($documentId, $sessionId, $sessionToken, $version, $steps, $token);
+	public function push(int $documentId, int $sessionId, string $sessionToken, int $version, array $steps, string $awareness, string $token): DataResponse {
+		return $this->apiService->push($documentId, $sessionId, $sessionToken, $version, $steps, $awareness, $token);
 	}
 
 	/**

lib/Controller/SessionController.php

@@ -69,9 +69,9 @@ public function close(int $documentId, int $sessionId, string $sessionToken): Da
 	 * @NoAdminRequired
 	 * @PublicPage
 	 */
-	public function push(int $documentId, int $sessionId, string $sessionToken, int $version, array $steps): DataResponse {
+	public function push(int $documentId, int $sessionId, string $sessionToken, int $version, array $steps, string $awareness): DataResponse {
 		$this->loginSessionUser($documentId, $sessionId, $sessionToken);
-		return $this->apiService->push($documentId, $sessionId, $sessionToken, $version, $steps);
+		return $this->apiService->push($documentId, $sessionId, $sessionToken, $version, $steps, $awareness);
 	}
 
 	/**

lib/Db/Session.php

@@ -35,6 +35,8 @@
  * @method void setColor(string $color)
  * @method string|null getGuestName()
  * @method void setGuestName(string $guestName)
+ * @method string|null getAwarenessMessage()
+ * @method void setLastAwarenessMessage(string $message)
  * @method int getLastContact()
  * @method void setLastContact(int $getTime)
  * @method int getDocumentId()
@@ -46,6 +48,7 @@ class Session extends Entity implements JsonSerializable {
 	protected string $token = '';
 	protected string $color = '';
 	protected ?string $guestName = null;
+	protected ?string $lastAwarenessMessage = '';
 	protected int $lastContact = 0;
 	protected int $documentId = 0;
 
@@ -61,6 +64,7 @@ public function jsonSerialize(): array {
 			'userId' => $this->userId,
 			'token' => $this->token,
 			'color' => $this->color,
+			'lastAwarenessMessage' => $this->lastAwarenessMessage,
 			'lastContact' => $this->lastContact,
 			'guestName' => $this->guestName,
 			'documentId' => $this->documentId,

lib/Db/SessionMapper.php

@@ -62,7 +62,7 @@ public function find($documentId, $sessionId, $token): Session {
 
 	public function findAll($documentId) {
 		$qb = $this->db->getQueryBuilder();
-		$qb->select('id', 'color', 'document_id', 'last_contact', 'user_id', 'guest_name')
+		$qb->select('id', 'color', 'document_id', 'last_awareness_message', 'last_contact', 'user_id', 'guest_name')
 			->from($this->getTableName())
 			->where($qb->expr()->eq('document_id', $qb->createNamedParameter($documentId)));
 
@@ -71,7 +71,7 @@ public function findAll($documentId) {
 
 	public function findAllActive($documentId) {
 		$qb = $this->db->getQueryBuilder();
-		$qb->select('id', 'color', 'document_id', 'last_contact', 'user_id', 'guest_name')
+		$qb->select('id', 'color', 'document_id', 'last_awareness_message', 'last_contact', 'user_id', 'guest_name')
 			->from($this->getTableName())
 			->where($qb->expr()->eq('document_id', $qb->createNamedParameter($documentId)))
 			->andWhere($qb->expr()->gt('last_contact', $qb->createNamedParameter(time() - SessionService::SESSION_VALID_TIME)));
@@ -81,7 +81,7 @@ public function findAllActive($documentId) {
 
 	public function findAllInactive() {
 		$qb = $this->db->getQueryBuilder();
-		$qb->select('id', 'color', 'document_id', 'last_contact', 'user_id', 'guest_name')
+		$qb->select('id', 'color', 'document_id', 'last_awareness_message', 'last_contact', 'user_id', 'guest_name')
 			->from($this->getTableName())
 			->where($qb->expr()->lt('last_contact', $qb->createNamedParameter(time() - SessionService::SESSION_VALID_TIME)));
 

lib/Migration/Version030701Date20230207131313.php

@@ -0,0 +1,38 @@
+<?php
+
+declare(strict_types=1);
+
+namespace OCA\Text\Migration;
+
+use Closure;
+use OCP\DB\ISchemaWrapper;
+use OCP\Migration\IOutput;
+use OCP\Migration\SimpleMigrationStep;
+
+/**
+ * Auto-generated migration step: Please modify to your needs!
+ */
+class Version030701Date20230207131313 extends SimpleMigrationStep {
+
+	/**
+	 * @param IOutput $output
+	 * @param Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper`
+	 * @param array $options
+	 * @return null|ISchemaWrapper
+	 */
+	public function changeSchema(IOutput $output, Closure $schemaClosure, array $options) {
+		/** @var ISchemaWrapper $schema */
+		$schema = $schemaClosure();
+
+		$table = $schema->getTable('text_sessions');
+		if (!$table->hasColumn('last_awareness_message')) {
+			$table->addColumn('last_awareness_message', \OCP\DB\Types::TEXT, [
+				'notnull' => false,
+				'default' => ''
+			]);
+			return $schema;
+		}
+
+		return null;
+	}
+}

lib/Service/ApiService.php

@@ -184,11 +184,15 @@ public function close($documentId, $sessionId, $sessionToken): DataResponse {
 	 * @throws NotFoundException
 	 * @throws DoesNotExistException
 	 */
-	public function push($documentId, $sessionId, $sessionToken, $version, $steps, $token = null): DataResponse {
+	public function push($documentId, $sessionId, $sessionToken, $version, $steps, $awareness, $token = null): DataResponse {
 		if (!$this->sessionService->isValidSession($documentId, $sessionId, $sessionToken)) {
 			return new DataResponse([], 403);
 		}
 		$session = $this->sessionService->getSession($documentId, $sessionId, $sessionToken);
+		$this->sessionService->updateSessionAwareness($documentId, $sessionId, $sessionToken, $awareness);
+		if (empty($steps)) {
+			return new DataResponse([]);
+		}
 		$file = $this->documentService->getFileForSession($session, $token);
 		if (!$this->documentService->isReadOnly($file, $token)) {
 			try {

lib/Service/SessionService.php

@@ -230,6 +230,23 @@ public function updateSession(int $documentId, int $sessionId, string $sessionTo
 		return $this->sessionMapper->update($session);
 	}
 
+	/**
+	 * @param $documentId
+	 * @param $sessionId
+	 * @param $sessionToken
+	 * @param $message
+	 * @return Session
+	 * @throws DoesNotExistException
+	 */
+	public function updateSessionAwareness(int $documentId, int $sessionId, string $sessionToken, string $message): Session {
+		$session = $this->sessionMapper->find($documentId, $sessionId, $sessionToken);
+		if (empty($message)) {
+			return $session;
+		}
+		$session->setLastAwarenessMessage($message);
+		return $this->sessionMapper->update($session);
+	}
+
 	private function getColorForGuestName(string $guestName = null): string {
 		$guestName = $this->userId ?? $guestName;
 		$uniqueGuestId = !empty($guestName) ? $guestName : $this->secureRandom->generate(12);

src/services/PollingBackend.js

@@ -171,6 +171,7 @@ class PollingBackend {
 		}
 
 		this.#syncService.emit('change', { document, sessions })
+		this.#syncService._receiveSteps(data)
 
 		if (data.steps.length === 0) {
 			if (!this.#initialLoadingFinished) {
@@ -192,7 +193,6 @@ class PollingBackend {
 			return
 		}
 
-		this.#syncService._receiveSteps(data)
 		this.#forcedSave = false
 		if (this.#initialLoadingFinished) {
 			this.resetRefetchTimer()

src/services/SessionApi.js

@@ -106,12 +106,13 @@ export class Connection {
 		})
 	}
 
-	push({ steps, version }) {
+	push({ steps, version, awareness }) {
 		return axios.post(this.#url('session/push'), {
 			...this.#defaultParams,
 			filePath: this.#options.filePath,
 			steps,
 			version,
+			awareness,
 		})
 	}
 

src/services/SyncService.js

@@ -158,15 +158,14 @@ class SyncService {
 			})
 	}
 
-	stepsSince(version) {
-		return {
-			steps: this.steps.slice(version),
-			clientIDs: this.stepClientIDs.slice(version),
-		}
-	}
-
-	_receiveSteps({ steps, document }) {
-		const newSteps = []
+	_receiveSteps({ steps, document, sessions }) {
+		const awareness = sessions
+			.filter(s => s.lastAwarenessMessage)
+			.map(s => {
+				return { step: s.lastAwarenessMessage, clientId: s.clientId }
+			})
+		const newSteps = awareness
+		this.steps = [...this.steps, ...awareness.map(s => s.step)]
 		for (let i = 0; i < steps.length; i++) {
 			const singleSteps = steps[i].data
 			if (this.version < steps[i].version) {

src/services/WebSocketPolyfill.js

@@ -88,19 +88,33 @@ export default function initWebSocketPolyfill(syncService, fileId, initialSessio
 		}
 
 		#initiateSending() {
-			let steps
+			let outbox
 			syncService.sendSteps(() => {
-				steps = this.#queue.map(s => encodeArrayBuffer(s))
+				outbox = this.#queue
+				const data = {
+					steps: this.#steps,
+					awareness: this.#awareness,
+					version: this.#version,
+				}
 				this.#queue = []
-				const version = this.#version
-				logger.debug('sending steps ', { version, steps })
-				return { version, steps }
+				logger.debug('sending steps ', data)
+				return data
 			})?.catch(() => {
-				// try again
-				this.send(...steps)
+				// try to send the steps again
+				this.#queue = [...outbox, ...this.#queue]
 			})
 		}
 
+		get #steps() {
+			return this.#queue.map(s => encodeArrayBuffer(s))
+				.filter(s => s < 'AQ')
+		}
+
+		get #awareness() {
+			return this.#queue.map(s => encodeArrayBuffer(s))
+				.findLast(s => s > 'AQ') || ''
+		}
+
 		close() {
 			Object.entries(this.#handlers)
 				.forEach(([key, value]) => syncService.off(key, value))