Feat/sync truncation2

apps/dav/appinfo/v1/carddav.php

@@ -57,12 +57,13 @@
 	'principals/'
 );
 $db = Server::get(IDBConnection::class);
 $cardDavBackend = new CardDavBackend(
 	$db,
 	$principalBackend,
 	Server::get(IUserManager::class),
 	Server::get(IEventDispatcher::class),
 	Server::get(\OCA\DAV\CardDAV\Sharing\Backend::class),
+	Server::get(IConfig::class),
 );
 
 $debugging = Server::get(IConfig::class)->getSystemValue('debug', false);

apps/dav/lib/CardDAV/AddressBook.php

@@ -8,7 +8,6 @@
 namespace OCA\DAV\CardDAV;
 
 use OCA\DAV\DAV\Sharing\IShareable;
-use OCA\DAV\Exception\UnsupportedLimitOnInitialSyncException;
 use OCP\DB\Exception;
 use OCP\IL10N;
 use OCP\Server;
@@ -234,9 +233,6 @@ private function canWrite(): bool {
 	}
 
 	public function getChanges($syncToken, $syncLevel, $limit = null) {
-		if (!$syncToken && $limit) {
-			throw new UnsupportedLimitOnInitialSyncException();
-		}
 
 		return parent::getChanges($syncToken, $syncLevel, $limit);
 	}

apps/dav/lib/CardDAV/CardDavBackend.php

@@ -23,9 +23,11 @@
 use OCP\DB\Exception;
 use OCP\DB\QueryBuilder\IQueryBuilder;
 use OCP\EventDispatcher\IEventDispatcher;
+use OCP\IConfig;
 use OCP\IDBConnection;
 use OCP\IUserManager;
 use PDO;
+use Psr\Log\LoggerInterface;
 use Sabre\CardDAV\Backend\BackendInterface;
 use Sabre\CardDAV\Backend\SyncSupport;
 use Sabre\CardDAV\Plugin;
@@ -59,6 +61,8 @@ public function __construct(
 		private IUserManager $userManager,
 		private IEventDispatcher $dispatcher,
 		private Sharing\Backend $sharingBackend,
+		private LoggerInterface $logger,
+		private IConfig $config,
 	) {
 	}
 
@@ -850,6 +854,8 @@ public function deleteCard($addressBookId, $cardUri) {
 	 * @return array
 	 */
 	public function getChangesForAddressBook($addressBookId, $syncToken, $syncLevel, $limit = null) {
+		$maxLimit = $this->config->getSystemValueInt('carddav_sync_request_truncation', 50000);
+		$limit = ($limit === null) ? $maxLimit : min($limit, $maxLimit);
 		// Current synctoken
 		return $this->atomic(function () use ($addressBookId, $syncToken, $syncLevel, $limit) {
 			$qb = $this->db->getQueryBuilder();
@@ -872,10 +878,35 @@ public function getChangesForAddressBook($addressBookId, $syncToken, $syncLevel,
 				'modified' => [],
 				'deleted' => [],
 			];
-
-			if ($syncToken) {
+			if (str_starts_with($syncToken, 'init_')) {
+				$syncValues = explode('_', $syncToken);
+				$lastID = $syncValues[1];
+				$initialSyncToken = $syncValues[2];
+				$qb = $this->db->getQueryBuilder();
+				$qb->select('id', 'uri')
+					->from('cards')
+					->where(
+						$qb->expr()->andX(
+							$qb->expr()->eq('addressbookid', $qb->createNamedParameter($addressBookId)),
+							$qb->expr()->gt('id', $qb->createNamedParameter($lastID)))
+					)->orderBy('id')
+					->setMaxResults($limit);
+				$stmt = $qb->executeQuery();
+				$values = $stmt->fetchAll(\PDO::FETCH_ASSOC);
+				$stmt->closeCursor();
+				if (count($values) === 0) {
+					$result['syncToken'] = $initialSyncToken;
+					$result['result_truncated'] = false;
+					$result['added'] = [];
+				} else {
+					$lastID = end($values)['id'];
+					$result['added'] = array_column($values, 'uri');
+					$result['syncToken'] = count($result['added']) === $limit ? "init_{$lastID}_$initialSyncToken" : $initialSyncToken ;
+					$result['result_truncated'] = count($result['added']) === $limit;
+				}
+			} elseif ($syncToken) {
 				$qb = $this->db->getQueryBuilder();
-				$qb->select('uri', 'operation')
+				$qb->select('uri', 'operation', 'synctoken')
 					->from('addressbookchanges')
 					->where(
 						$qb->expr()->andX(
@@ -885,20 +916,24 @@ public function getChangesForAddressBook($addressBookId, $syncToken, $syncLevel,
 						)
 					)->orderBy('synctoken');
 
-				if (is_int($limit) && $limit > 0) {
+				if ($limit > 0) {
 					$qb->setMaxResults($limit);
 				}
 
 				// Fetching all changes
 				$stmt = $qb->executeQuery();
+				$rowCount = $stmt->rowCount();
 
 				$changes = [];
+				$highestSyncToken = 0;
 
 				// This loop ensures that any duplicates are overwritten, only the
 				// last change on a node is relevant.
 				while ($row = $stmt->fetch(\PDO::FETCH_ASSOC)) {
 					$changes[$row['uri']] = $row['operation'];
+					$highestSyncToken = $row['synctoken'];
 				}
+
 				$stmt->closeCursor();
 
 				foreach ($changes as $uri => $operation) {
@@ -914,16 +949,38 @@ public function getChangesForAddressBook($addressBookId, $syncToken, $syncLevel,
 							break;
 					}
 				}
+
+				/*
+				 * The synctoken in oc_addressbooks is always the highest synctoken in oc_addressbookchanges for a given addressbook plus one (see addChange).
+				 *
+				 * For truncated results, it is expected that we return the highest token from the response, so the client can continue from the latest change.
+				 *
+				 * For non-truncated results, it is expected to return the currentToken. If we return the highest token, as with truncated results, the client will always think it is one change behind.
+				 *
+				 * Therefore, we differentiate between truncated and non-truncated results when returning the synctoken.
+				 */
+				if ($rowCount === $limit && $highestSyncToken < $currentToken) {
+					$result['syncToken'] = $highestSyncToken;
+				}
 			} else {
 				$qb = $this->db->getQueryBuilder();
-				$qb->select('uri')
+				$qb->select('id', 'uri')
 					->from('cards')
 					->where(
 						$qb->expr()->eq('addressbookid', $qb->createNamedParameter($addressBookId))
 					);
 				// No synctoken supplied, this is the initial sync.
+				$qb->setMaxResults($limit);
 				$stmt = $qb->executeQuery();
-				$result['added'] = $stmt->fetchAll(\PDO::FETCH_COLUMN);
+				$values = $stmt->fetchAll(\PDO::FETCH_ASSOC);
+				$lastID = $values[array_key_last($values)]['id'];
+				if (count(array_values($values)) >= $limit) {
+					$result['syncToken'] = 'init_' . $lastID . '_' . $currentToken;
+					$result['result_truncated'] = true;
+				}
+
+				$result['added'] = array_column($values, 'uri');
+
 				$stmt->closeCursor();
 			}
 			return $result;

apps/dav/lib/CardDAV/SyncService.php

@@ -22,6 +22,7 @@
 use Sabre\DAV\Xml\Response\MultiStatus;
 use Sabre\DAV\Xml\Service;
 use Sabre\VObject\Reader;
+use Sabre\Xml\ParseException;
 use function is_null;
 
 class SyncService {
@@ -43,9 +44,10 @@
 	}
 
 	/**
+	 * @psalm-return list{0: ?string, 1: boolean}
 	 * @throws \Exception
 	 */
-	public function syncRemoteAddressBook(string $url, string $userName, string $addressBookUrl, string $sharedSecret, ?string $syncToken, string $targetBookHash, string $targetPrincipal, array $targetProperties): string {
+	public function syncRemoteAddressBook(string $url, string $userName, string $addressBookUrl, string $sharedSecret, ?string $syncToken, string $targetBookHash, string $targetPrincipal, array $targetProperties): array {
 		// 1. create addressbook
 		$book = $this->ensureSystemAddressBookExists($targetPrincipal, $targetBookHash, $targetProperties);
 		$addressBookId = $book['id'];
@@ -83,7 +85,10 @@
 			}
 		}
 
-		return $response['token'];
+		return [
+			$response['token'],
+			$response['truncated'],
+		];
 	}
 
 	/**
@@ -127,7 +132,7 @@
 
 	private function prepareUri(string $host, string $path): string {
 		/*
-		 * The trailing slash is important for merging the uris together.
+		 * The trailing slash is important for merging the uris.
 		 *
 		 * $host is stored in oc_trusted_servers.url and usually without a trailing slash.
 		 *
@@ -158,9 +163,11 @@
 	}
 
 	/**
+	 * @return array{response: array{string, array<array-key, mixed}, token: ?string, truncated: boolean}
 	 * @throws ClientExceptionInterface
+	 * @throws ParseException
 	 */
 	protected function requestSyncReport(string $url, string $userName, string $addressBookUrl, string $sharedSecret, ?string $syncToken): array {
 		$client = $this->clientService->newClient();
 		$uri = $this->prepareUri($url, $addressBookUrl);
 
@@ -181,7 +188,7 @@
 		$body = $response->getBody();
 		assert(is_string($body));
 
-		return $this->parseMultiStatus($body);
+		return $this->parseMultiStatus($body, $addressBookUrl);
 	}
 
 	protected function download(string $url, string $userName, string $sharedSecret, string $resourcePath): string {
@@ -219,22 +226,50 @@
 	}
 
 	/**
-	 * @param string $body
-	 * @return array
-	 * @throws \Sabre\Xml\ParseException
+	 * @return array{response: array{string, array<array-key, mixed}, token: ?string, truncated: boolean}
+	 * @throws ParseException
 	 */
-	private function parseMultiStatus($body) {
-		$xml = new Service();
-
+	private function parseMultiStatus(string $body, string $addressBookUrl): array {
 		/** @var MultiStatus $multiStatus */
-		$multiStatus = $xml->expect('{DAV:}multistatus', $body);
+		$multiStatus = (new Service())->expect('{DAV:}multistatus', $body);
 
 		$result = [];
+		$truncated = false;
+
 		foreach ($multiStatus->getResponses() as $response) {
-			$result[$response->getHref()] = $response->getResponseProperties();
+			$href = $response->getHref();
+			if ($response->getHttpStatus() === '507' && $this->isResponseForRequestUri($href, $addressBookUrl)) {
+				$truncated = true;
+			} else {
+				$result[$response->getHref()] = $response->getResponseProperties();
+			}
 		}
 
-		return ['response' => $result, 'token' => $multiStatus->getSyncToken()];
+		return ['response' => $result, 'token' => $multiStatus->getSyncToken(), 'truncated' => $truncated];
+	}
+
+	/**
+	 * Determines whether the provided response URI corresponds to the given request URI.
+	 */
+	private function isResponseForRequestUri(string $responseUri, string $requestUri): bool {
+		/*
+		 * Example response uri:
+		 *
+		 * /remote.php/dav/addressbooks/system/system/system/
+		 * /cloud/remote.php/dav/addressbooks/system/system/system/ (when installed in a subdirectory)
+		 *
+		 * Example request uri:
+		 *
+		 * remote.php/dav/addressbooks/system/system/system
+		 *
+		 * References:
+		 * https://github.com/nextcloud/3rdparty/blob/e0a509739b13820f0a62ff9cad5d0fede00e76ee/sabre/dav/lib/DAV/Sync/Plugin.php#L172-L174
+		 * https://github.com/nextcloud/server/blob/b40acb34a39592070d8455eb91c5364c07928c50/apps/federation/lib/SyncFederationAddressBooks.php#L41
+		 */
+		return str_ends_with(
+			rtrim($responseUri, '/'),
+			rtrim($requestUri, '/')
+		);
 	}
 
 	/**

apps/dav/lib/CardDAV/SystemAddressbook.php

@@ -8,7 +8,6 @@
  */
 namespace OCA\DAV\CardDAV;
 
-use OCA\DAV\Exception\UnsupportedLimitOnInitialSyncException;
 use OCA\Federation\TrustedServers;
 use OCP\Accounts\IAccountManager;
 use OCP\IConfig;
@@ -212,14 +211,7 @@ public function getChild($name): Card {
 		}
 		return new Card($this->carddavBackend, $this->addressBookInfo, $obj);
 	}
-
-	/**
-	 * @throws UnsupportedLimitOnInitialSyncException
-	 */
 	public function getChanges($syncToken, $syncLevel, $limit = null) {
-		if (!$syncToken && $limit) {
-			throw new UnsupportedLimitOnInitialSyncException();
-		}
 
 		if (!$this->carddavBackend instanceof SyncSupport) {
 			return null;

apps/dav/lib/RootCollection.php

@@ -132,6 +132,8 @@ public function __construct() {
 		);
 
 		$contactsSharingBackend = Server::get(\OCA\DAV\CardDAV\Sharing\Backend::class);
+		$logger = Server::get(\Psr\Log\LoggerInterface::class);
+		$config = Server::get(IConfig::class);
 
 		$pluginManager = new PluginManager(\OC::$server, Server::get(IAppManager::class));
 		$usersCardDavBackend = new CardDavBackend(
@@ -140,6 +142,8 @@ public function __construct() {
 			$userManager,
 			$dispatcher,
 			$contactsSharingBackend,
+			$logger,
+			$config
 		);
 		$usersAddressBookRoot = new AddressBookRoot($userPrincipalBackend, $usersCardDavBackend, $pluginManager, $userSession->getUser(), $groupManager, 'principals/users');
 		$usersAddressBookRoot->disableListing = $disableListing;
@@ -150,6 +154,8 @@ public function __construct() {
 			$userManager,
 			$dispatcher,
 			$contactsSharingBackend,
+			$logger,
+			$config
 		);
 		$systemAddressBookRoot = new AddressBookRoot(new SystemPrincipalBackend(), $systemCardDavBackend, $pluginManager, $userSession->getUser(), $groupManager, 'principals/system');
 		$systemAddressBookRoot->disableListing = $disableListing;

apps/dav/lib/Server.php

@@ -175,7 +175,6 @@ public function __construct(
 
 		$this->server->addPlugin(new ExceptionLoggerPlugin('webdav', $logger));
 		$this->server->addPlugin(new LockPlugin());
-		$this->server->addPlugin(new \Sabre\DAV\Sync\Plugin());
 
 		// acl
 		$acl = new DavAclPlugin();