Skip to content

Use search to get versions list from cache for expiry #27637

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
juliusknorr merged 2 commits into from
Aug 19, 2021
Merged

Use search to get versions list from cache for expiry #27637

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d20aad3
test for version expire
icewind1991 Jun 22, 2021
5f101d9
use search to find versions to expire
icewind1991 Aug 18, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 43 additions & 18 deletions apps/files_versions/lib/Storage.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,20 +37,29 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>
*
*/

namespace OCA\Files_Versions;

use OC\Files\Search\SearchBinaryOperator;
use OC\Files\Search\SearchComparison;
use OC\Files\Search\SearchQuery;
use OC_User;
use OC\Files\Filesystem;
use OC\Files\View;
use OCA\Files_Versions\AppInfo\Application;
use OCA\Files_Versions\Command\Expire;
use OCA\Files_Versions\Events\CreateVersionEvent;
use OCA\Files_Versions\Versions\IVersionManager;
use OCP\Files\FileInfo;
use OCP\Files\Folder;
use OCP\Files\IRootFolder;
use OCP\Files\Node;
use OCP\Command\IBus;
use OCP\EventDispatcher\IEventDispatcher;
use OCP\Files\IMimeTypeDetector;
use OCP\Files\IRootFolder;
use OCP\Files\NotFoundException;
use OCP\Files\Search\ISearchBinaryOperator;
use OCP\Files\Search\ISearchComparison;
use OCP\Files\StorageNotAvailableException;
use OCP\IURLGenerator;
use OCP\IUser;
Expand Down Expand Up @@ -495,38 +504,54 @@ public static function getVersions($uid, $filename, $userFullPath = '') {

/**
* Expire versions that older than max version retention time
*
* @param string $uid
*/
public static function expireOlderThanMaxForUser($uid) {
/** @var IRootFolder $root */
$root = \OC::$server->get(IRootFolder::class);
try {
/** @var Folder $versionsRoot */
$versionsRoot = $root->get('/' . $uid . '/files_versions');
} catch (NotFoundException $e) {
return;
}

$expiration = self::getExpiration();
$threshold = $expiration->getMaxAgeAsTimestamp();
$versions = self::getAllVersions($uid);
if (!$threshold || empty($versions['all'])) {
if (!$threshold) {
return;
}

$toDelete = [];
foreach (array_reverse($versions['all']) as $key => $version) {
if ((int)$version['version'] < $threshold) {
$toDelete[$key] = $version;
} else {
//Versions are sorted by time - nothing mo to iterate.
break;
}
}
$allVersions = $versionsRoot->search(new SearchQuery(
new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_NOT, [
new SearchComparison(ISearchComparison::COMPARE_EQUAL, 'mimetype', FileInfo::MIMETYPE_FOLDER),
]),
0,
0,
[]
));

$view = new View('/' . $uid . '/files_versions');
if (!empty($toDelete)) {
foreach ($toDelete as $version) {
\OC_Hook::emit('\OCP\Versions', 'preDelete', ['path' => $version['path'].'.v'.$version['version'], 'trigger' => self::DELETE_TRIGGER_RETENTION_CONSTRAINT]);
self::deleteVersion($view, $version['path'] . '.v' . $version['version']);
\OC_Hook::emit('\OCP\Versions', 'delete', ['path' => $version['path'].'.v'.$version['version'], 'trigger' => self::DELETE_TRIGGER_RETENTION_CONSTRAINT]);
/** @var Node[] $versions */
$versions = array_filter($allVersions, function (Node $info) use ($threshold) {
$versionsBegin = strrpos($info->getName(), '.v');
if ($versionsBegin === false) {
return false;
}
$version = (int)substr($info->getName(), $versionsBegin + 2);
return $version < $threshold;
});

foreach ($versions as $version) {
\OC_Hook::emit('\OCP\Versions', 'preDelete', ['path' => $version->getInternalPath(), 'trigger' => self::DELETE_TRIGGER_RETENTION_CONSTRAINT]);
$version->delete();
\OC_Hook::emit('\OCP\Versions', 'delete', ['path' => $version->getInternalPath(), 'trigger' => self::DELETE_TRIGGER_RETENTION_CONSTRAINT]);
}
}

/**
* translate a timestamp into a string like "5 days ago"
*
* @param int $timestamp
* @return string for example "5 days ago"
*/
Expand Down
116 changes: 116 additions & 0 deletions apps/files_versions/tests/StorageTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
<?php

declare(strict_types=1);
/**
* @copyright Copyright (c) 2021 Robin Appelman <[email protected]>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/

namespace OCA\files_versions\tests;

use OCA\Files_Versions\Expiration;
use OCA\Files_Versions\Hooks;
use OCA\Files_Versions\Storage;
use OCP\Files\IRootFolder;
use OCP\Files\NotFoundException;
use Test\TestCase;
use Test\Traits\UserTrait;

/**
* @group DB
*/
class StorageTest extends TestCase {
use UserTrait;

private $versionsRoot;
private $userFolder;
private $expireTimestamp = 10;

protected function setUp(): void {
parent::setUp();

$expiration = $this->createMock(Expiration::class);
$expiration->method('getMaxAgeAsTimestamp')
->willReturnCallback(function () {
return $this->expireTimestamp;
});
$this->overwriteService(Expiration::class, $expiration);

Hooks::connectHooks();

$this->createUser('version_test', '');
$this->loginAsUser('version_test');
/** @var IRootFolder $root */
$root = \OC::$server->get(IRootFolder::class);
$this->userFolder = $root->getUserFolder('version_test');
}


protected function createPastFile(string $path, int $mtime) {
try {
$file = $this->userFolder->get($path);
} catch (NotFoundException $e) {
$file = $this->userFolder->newFile($path);
}
$file->putContent((string)$mtime);
$file->touch($mtime);
}

public function testExpireMaxAge() {
$this->userFolder->newFolder('folder1');
$this->userFolder->newFolder('folder1/sub1');
$this->userFolder->newFolder('folder2');

$this->createPastFile('file1', 100);
$this->createPastFile('file1', 500);
$this->createPastFile('file1', 900);

$this->createPastFile('folder1/file2', 100);
$this->createPastFile('folder1/file2', 200);
$this->createPastFile('folder1/file2', 300);

$this->createPastFile('folder1/sub1/file3', 400);
$this->createPastFile('folder1/sub1/file3', 500);
$this->createPastFile('folder1/sub1/file3', 600);

$this->createPastFile('folder2/file4', 100);
$this->createPastFile('folder2/file4', 600);
$this->createPastFile('folder2/file4', 800);

$this->assertCount(2, Storage::getVersions('version_test', 'file1'));
$this->assertCount(2, Storage::getVersions('version_test', 'folder1/file2'));
$this->assertCount(2, Storage::getVersions('version_test', 'folder1/sub1/file3'));
$this->assertCount(2, Storage::getVersions('version_test', 'folder2/file4'));

$this->expireTimestamp = 150;
Storage::expireOlderThanMaxForUser('version_test');

$this->assertCount(1, Storage::getVersions('version_test', 'file1'));
$this->assertCount(1, Storage::getVersions('version_test', 'folder1/file2'));
$this->assertCount(2, Storage::getVersions('version_test', 'folder1/sub1/file3'));
$this->assertCount(1, Storage::getVersions('version_test', 'folder2/file4'));

$this->expireTimestamp = 550;
Storage::expireOlderThanMaxForUser('version_test');

$this->assertCount(0, Storage::getVersions('version_test', 'file1'));
$this->assertCount(0, Storage::getVersions('version_test', 'folder1/file2'));
$this->assertCount(0, Storage::getVersions('version_test', 'folder1/sub1/file3'));
$this->assertCount(1, Storage::getVersions('version_test', 'folder2/file4'));
}
}