A dav plugin to allow pagination of multipart responses

Signed-off-by: Robin Appelman <[email protected]>

Author: icewind1991

3rdparty

@@ -5,7 +5,7 @@
 	<name>WebDAV</name>
 	<summary>WebDAV endpoint</summary>
 	<description>WebDAV endpoint</description>
-	<version>1.10.0</version>
+	<version>1.10.1</version>
 	<licence>agpl</licence>
 	<author>owncloud.org</author>
 	<namespace>DAV</namespace>
@@ -23,6 +23,7 @@
 		<job>OCA\DAV\BackgroundJob\CleanupDirectLinksJob</job>
 		<job>OCA\DAV\BackgroundJob\UpdateCalendarResourcesRoomsBackgroundJob</job>
 		<job>OCA\DAV\BackgroundJob\CleanupInvitationTokenJob</job>
+		<job>OCA\DAV\BackgroundJob\CleanupPaginateCacheJob</job>
 	</background-jobs>
 
 	<repair-steps>

apps/dav/appinfo/info.xml

@@ -13,6 +13,7 @@
     'OCA\\DAV\\Avatars\\RootCollection' => $baseDir . '/../lib/Avatars/RootCollection.php',
     'OCA\\DAV\\BackgroundJob\\CleanupDirectLinksJob' => $baseDir . '/../lib/BackgroundJob/CleanupDirectLinksJob.php',
     'OCA\\DAV\\BackgroundJob\\CleanupInvitationTokenJob' => $baseDir . '/../lib/BackgroundJob/CleanupInvitationTokenJob.php',
+    'OCA\\DAV\\BackgroundJob\\CleanupPaginateCacheJob' => $baseDir . '/../lib/BackgroundJob/CleanupPaginateCacheJob.php',
     'OCA\\DAV\\BackgroundJob\\GenerateBirthdayCalendarBackgroundJob' => $baseDir . '/../lib/BackgroundJob/GenerateBirthdayCalendarBackgroundJob.php',
     'OCA\\DAV\\BackgroundJob\\RefreshWebcalJob' => $baseDir . '/../lib/BackgroundJob/RefreshWebcalJob.php',
     'OCA\\DAV\\BackgroundJob\\RegisterRegenerateBirthdayCalendars' => $baseDir . '/../lib/BackgroundJob/RegisterRegenerateBirthdayCalendars.php',
@@ -178,6 +179,10 @@
     'OCA\\DAV\\Migration\\Version1008Date20181105110300' => $baseDir . '/../lib/Migration/Version1008Date20181105110300.php',
     'OCA\\DAV\\Migration\\Version1008Date20181105112049' => $baseDir . '/../lib/Migration/Version1008Date20181105112049.php',
     'OCA\\DAV\\Migration\\Version1008Date20181114084440' => $baseDir . '/../lib/Migration/Version1008Date20181114084440.php',
+    'OCA\\DAV\\Migration\\Version1009Date20181108161232' => $baseDir . '/../lib/Migration/Version1009Date20181108161232.php',
+    'OCA\\DAV\\Paginate\\LimitedCopyIterator' => $baseDir . '/../lib/Paginate/LimitedCopyIterator.php',
+    'OCA\\DAV\\Paginate\\PaginateCache' => $baseDir . '/../lib/Paginate/PaginateCache.php',
+    'OCA\\DAV\\Paginate\\PaginatePlugin' => $baseDir . '/../lib/Paginate/PaginatePlugin.php',
     'OCA\\DAV\\Provisioning\\Apple\\AppleProvisioningNode' => $baseDir . '/../lib/Provisioning/Apple/AppleProvisioningNode.php',
     'OCA\\DAV\\Provisioning\\Apple\\AppleProvisioningPlugin' => $baseDir . '/../lib/Provisioning/Apple/AppleProvisioningPlugin.php',
     'OCA\\DAV\\RootCollection' => $baseDir . '/../lib/RootCollection.php',

apps/dav/composer/composer/autoload_classmap.php

@@ -28,6 +28,7 @@ class ComposerStaticInitDAV
         'OCA\\DAV\\Avatars\\RootCollection' => __DIR__ . '/..' . '/../lib/Avatars/RootCollection.php',
         'OCA\\DAV\\BackgroundJob\\CleanupDirectLinksJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/CleanupDirectLinksJob.php',
         'OCA\\DAV\\BackgroundJob\\CleanupInvitationTokenJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/CleanupInvitationTokenJob.php',
+        'OCA\\DAV\\BackgroundJob\\CleanupPaginateCacheJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/CleanupPaginateCacheJob.php',
         'OCA\\DAV\\BackgroundJob\\GenerateBirthdayCalendarBackgroundJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/GenerateBirthdayCalendarBackgroundJob.php',
         'OCA\\DAV\\BackgroundJob\\RefreshWebcalJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/RefreshWebcalJob.php',
         'OCA\\DAV\\BackgroundJob\\RegisterRegenerateBirthdayCalendars' => __DIR__ . '/..' . '/../lib/BackgroundJob/RegisterRegenerateBirthdayCalendars.php',
@@ -193,6 +194,10 @@ class ComposerStaticInitDAV
         'OCA\\DAV\\Migration\\Version1008Date20181105110300' => __DIR__ . '/..' . '/../lib/Migration/Version1008Date20181105110300.php',
         'OCA\\DAV\\Migration\\Version1008Date20181105112049' => __DIR__ . '/..' . '/../lib/Migration/Version1008Date20181105112049.php',
         'OCA\\DAV\\Migration\\Version1008Date20181114084440' => __DIR__ . '/..' . '/../lib/Migration/Version1008Date20181114084440.php',
+        'OCA\\DAV\\Migration\\Version1009Date20181108161232' => __DIR__ . '/..' . '/../lib/Migration/Version1009Date20181108161232.php',
+        'OCA\\DAV\\Paginate\\LimitedCopyIterator' => __DIR__ . '/..' . '/../lib/Paginate/LimitedCopyIterator.php',
+        'OCA\\DAV\\Paginate\\PaginateCache' => __DIR__ . '/..' . '/../lib/Paginate/PaginateCache.php',
+        'OCA\\DAV\\Paginate\\PaginatePlugin' => __DIR__ . '/..' . '/../lib/Paginate/PaginatePlugin.php',
         'OCA\\DAV\\Provisioning\\Apple\\AppleProvisioningNode' => __DIR__ . '/..' . '/../lib/Provisioning/Apple/AppleProvisioningNode.php',
         'OCA\\DAV\\Provisioning\\Apple\\AppleProvisioningPlugin' => __DIR__ . '/..' . '/../lib/Provisioning/Apple/AppleProvisioningPlugin.php',
         'OCA\\DAV\\RootCollection' => __DIR__ . '/..' . '/../lib/RootCollection.php',

apps/dav/composer/composer/autoload_static.php

@@ -0,0 +1,40 @@
+<?php
+/**
+ * @copyright Copyright (c) 2018 Robin Appelman <[email protected]>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OCA\DAV\BackgroundJob;
+
+use OC\BackgroundJob\Job;
+use OCA\DAV\Paginate\PaginateCache;
+
+class CleanupPaginateCacheJob extends Job {
+
+	/** @var PaginateCache */
+	private $cache;
+
+	public function __construct(PaginateCache $cache) {
+		$this->cache = $cache;
+	}
+
+	public function run($argument) {
+		$this->cache->cleanup();
+	}
+
+}

apps/dav/lib/BackgroundJob/CleanupPaginateCacheJob.php

@@ -0,0 +1,55 @@
+<?php
+namespace OCA\DAV\Migration;
+
+use Doctrine\DBAL\Types\Type;
+use OCP\DB\ISchemaWrapper;
+use OCP\Migration\SimpleMigrationStep;
+use OCP\Migration\IOutput;
+
+class Version1009Date20181108161232 extends SimpleMigrationStep {
+	public function name(): string {
+		return 'Add dav_page_cache table';
+	}
+
+	public function description(): string {
+		return 'Add table to cache webdav multistatus responses for pagination purpose';
+	}
+
+	public function changeSchema(IOutput $output, \Closure $schemaClosure, array $options) {
+		/** @var ISchemaWrapper $schema */
+		$schema = $schemaClosure();
+
+		if (!$schema->hasTable('dav_page_cache')) {
+			$table = $schema->createTable('dav_page_cache');
+
+			$table->addColumn('id', Type::BIGINT, [
+				'autoincrement' => true
+			]);
+			$table->addColumn('url_hash', Type::STRING, [
+				'notnull' => true,
+				'length' => 32,
+			]);
+			$table->addColumn('token', Type::STRING, [
+				'notnull' => true,
+				'length' => 32
+			]);
+			$table->addColumn('result_index', Type::INTEGER, [
+				'notnull' => true
+			]);
+			$table->addColumn('result_value', TYPE::TEXT, [
+				'notnull' => false,
+			]);
+			$table->addColumn('insert_time', TYPE::BIGINT, [
+				'notnull' => false,
+			]);
+
+			$table->setPrimaryKey(['id'], 'dav_page_cache_id_index');
+			$table->addIndex(['token', 'url_hash'], 'dav_page_cache_token_url');
+			$table->addUniqueIndex(['token', 'url_hash', 'result_index'], 'dav_page_cache_url_index');
+			$table->addIndex(['result_index'], 'dav_page_cache_index');
+			$table->addIndex(['insert_time'], 'dav_page_cache_time');
+		}
+
+		return $schema;
+	}
+}

apps/dav/lib/Migration/Version1009Date20181108161232.php

@@ -0,0 +1,54 @@
+<?php
+/**
+ * @copyright Copyright (c) 2018 Robin Appelman <[email protected]>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OCA\DAV\Paginate;
+
+/**
+ * Save a copy of the first X items into a separate iterator
+ *
+ * this allows us to pass the iterator to the cache while keeping a copy
+ * of the first X items
+ */
+class LimitedCopyIterator extends \AppendIterator {
+	/** @var array */
+	private $copy;
+
+	public function __construct(\Traversable $iterator, int $count) {
+		parent::__construct();
+
+		if (!$iterator instanceof \Iterator) {
+			$iterator = new \IteratorIterator($iterator);
+		}
+		$iterator = new \NoRewindIterator($iterator);
+
+		while($iterator->valid() && count($this->copy) < $count) {
+			$this->copy[] = $iterator->current();
+			$iterator->next();
+		}
+
+		$this->append($this->getFirstItems());
+		$this->append($iterator);
+	}
+
+	public function getFirstItems(): \Iterator {
+		return new \ArrayIterator($this->copy);
+	}
+}

apps/dav/lib/Paginate/LimitedCopyIterator.php

@@ -0,0 +1,112 @@
+<?php
+/**
+ * @copyright Copyright (c) 2018 Robin Appelman <[email protected]>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OCA\DAV\Paginate;
+
+
+use OCP\AppFramework\Utility\ITimeFactory;
+use OCP\DB\QueryBuilder\IQueryBuilder;
+use OCP\IDBConnection;
+use OCP\Security\ISecureRandom;
+
+class PaginateCache {
+	const TTL = 3600;
+
+	/** @var IDBConnection */
+	private $database;
+	/** @var ISecureRandom */
+	private $random;
+	/** @var ITimeFactory */
+	private $timeFactory;
+
+	public function __construct(
+		IDBConnection $database,
+		ISecureRandom $random,
+		ITimeFactory $timeFactory
+	) {
+		$this->database = $database;
+		$this->random = $random;
+		$this->timeFactory = $timeFactory;
+	}
+
+	public function store(string $uri, \Iterator $items): array {
+		$token = $this->random->generate(32);
+		$now = $this->timeFactory->getTime();
+
+		$query = $this->database->getQueryBuilder();
+		$query->insert('dav_page_cache')
+			->values([
+				'url_hash' => $query->createNamedParameter(md5($uri), IQueryBuilder::PARAM_STR),
+				'token' => $query->createNamedParameter($token, IQueryBuilder::PARAM_STR),
+				'insert_time' => $query->createNamedParameter($now, IQueryBuilder::PARAM_INT),
+				'result_index' => $query->createParameter('index'),
+				'result_value' => $query->createParameter('value'),
+			]);
+
+		$count = 0;
+		foreach ($items as $item) {
+			$value = json_encode($item);
+			$query->setParameter('index', $count, IQueryBuilder::PARAM_INT);
+			$query->setParameter('value', $value);
+			$query->execute();
+			$count++;
+		}
+
+		return [$token, $count];
+	}
+
+	/**
+	 * @param string $url
+	 * @param string $token
+	 * @param int $offset
+	 * @param int $count
+	 * @return array|\Traversable
+	 */
+	public function get(string $url, string $token, int $offset, int $count) {
+		$query = $this->database->getQueryBuilder();
+		$query->select(['result_value'])
+			->from('dav_page_cache')
+			->where($query->expr()->eq('token', $query->createNamedParameter($token)))
+			->andWhere($query->expr()->eq('url_hash', $query->createNamedParameter(md5($url))))
+			->andWhere($query->expr()->gte('result_index', $query->createNamedParameter($offset, IQueryBuilder::PARAM_INT)))
+			->andWhere($query->expr()->lt('result_index', $query->createNamedParameter($offset + $count, IQueryBuilder::PARAM_INT)));
+
+		$result = $query->execute();
+		return array_map(function (string $entry) {
+			return json_decode($entry, true);
+		}, $result->fetchAll(\PDO::FETCH_COLUMN));
+	}
+
+	public function cleanup() {
+		$now = $this->timeFactory->getTime();
+
+		$query = $this->database->getQueryBuilder();
+		$query->delete('dav_page_cache')
+			->where($query->expr()->lt('insert_time', $query->createNamedParameter($now - self::TTL)));
+		$query->execute();
+	}
+
+	public function clear() {
+		$query = $this->database->getQueryBuilder();
+		$query->delete('dav_page_cache');
+		$query->execute();
+	}
+}