Skip to content

[stable9.1] Backport of filecache repair step OLD #29407

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
tomneedham wants to merge 15 commits into from
Closed

[stable9.1] Backport of filecache repair step OLD #29407

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
ebaa762
Add repair step to repair mismatch filecache paths
Jun 28, 2017
0980191
Repair detached filecache entries through occ files:scan
Jul 26, 2017
f79837d
Use DI and add more tests for storage filter
Aug 2, 2017
d6688a2
Run phase 2 of repair on gathered affected storages from phase 1
Aug 16, 2017
de8295c
occ files:scan --all --repair repairs all storages at once
Aug 16, 2017
ad2c23c
Improve output when repairing. Fix non-repair file:scan
tomneedham Aug 18, 2017
1ac999e
Add parallel non-corrupt storages to filecache repair step to check f…
tomneedham Sep 12, 2017
25edcf7
Test without parallel storages
tomneedham Sep 14, 2017
1a16772
Fix lastinsertid for postgres wehn reusing fileid
tomneedham Sep 14, 2017
38d8ce0
Try oracle tests without throwing exception for stray transactions
tomneedham Sep 15, 2017
3f3340e
Oracle hates empty strings
Sep 18, 2017
238728d
Workaround for Oracle trigger for fileid
Sep 19, 2017
bf3e4bf
Cleanup testcase and comments
tomneedham Sep 20, 2017
6f5f26a
Modify backported repair step for 9.1
tomneedham Nov 1, 2017
e81442e
Modify backported repair step for 9.1
tomneedham Nov 1, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Add parallel non-corrupt storages to filecache repair step to check f… 
…or isolation
  • Loading branch information
@tomneedham
tomneedham authored and Vincent Petry committed Nov 7, 2017
commit 1ac999e687d838670cbfd7d526c235b3bdac93e2
4 changes: 2 additions & 2 deletions apps/files/lib/Command/Scan.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -254,8 +254,8 @@ protected function execute(InputInterface $input, OutputInterface $output) {
// don't fix individually
$shouldRepairStoragesIndividually = false;
} else {
$output->writeln("<error>Repairing every storage individually is slower than repairing in bulk</error>");
$output->writeln("<error>To repair in bulk, please switch to single user mode first: occ maintenance:singleuser --on</error>");
$output->writeln("<comment>Repairing every storage individually is slower than repairing in bulk</comment>");
$output->writeln("<comment>To repair in bulk, please switch to single user mode first: occ maintenance:singleuser --on</comment>");
}
$users = $this->userManager->search('');
} else {
Expand Down
3 changes: 1 addition & 2 deletions lib/private/Files/Utils/Scanner.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,8 +195,7 @@ public function scan($dir = '') {
}

// if the home storage isn't writable then the scanner is run as the wrong user
$isHome = $storage->instanceOfStorage('\OC\Files\Storage\Home');
if ($isHome and
if ($storage->instanceOfStorage('\OC\Files\Storage\Home') and
(!$storage->isCreatable('') or !$storage->isCreatable('files'))
) {
if ($storage->file_exists('') or $storage->getCache()->inCache('')) {
Expand Down
4 changes: 3 additions & 1 deletion lib/private/Repair/RepairMismatchFileCachePath.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -232,7 +232,9 @@ private function reportAffectedStoragesParentIdWrongPath(IOutput $out) {

if (!empty($storageIds)) {
$out->warning('The file cache contains entries with invalid path values for the following storage numeric ids: ' . implode(' ', $storageIds));
$out->warning('Please run `occ files:scan --all --repair` to repair all affected storages');
$out->warning('Please run `occ files:scan --all --repair` to repair'
.'all affected storages or run `occ files:scan userid --repair for '
.'each user with affected storages');
}
}

Expand Down
221 changes: 213 additions & 8 deletions tests/lib/Repair/RepairMismatchFileCachePathTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -261,25 +261,23 @@ public function testRepairEntry($sourceStorageId, $targetStorageId, $targetDir,
$this->assertEquals('files/source/do_not_touch', $entry['path']);
$this->assertEquals(md5('files/source/do_not_touch'), $entry['path_hash']);

// root entry left alone
$entry = $this->getFileCacheEntry($rootId1);
$this->assertEquals(-1, $entry['parent']);
$this->assertEquals((string)$sourceStorageId, $entry['storage']);
$this->assertEquals('', $entry['path']);
$this->assertEquals(md5(''), $entry['path_hash']);
}

/**
* Test repair self referencing entries
*/
public function testRepairSelfReferencing() {
/**
* This is the storage tree that is created
* (alongside a normal storage, without corruption, but same structure)
*
*
* Self-referencing:
* - files/all_your_zombies (parent=fileid must be reparented)
*
* Referencing child one level:
* - files/ref_child1 (parent=fileid of the child)
* - files/ref_child1/child (parent=fileid of the child)
* - files/ref_child1/child
*
* Referencing child two levels:
* - files/ref_child2/ (parent=fileid of the child's child)
Expand All @@ -289,7 +287,13 @@ public function testRepairSelfReferencing() {
* Referencing child two levels detached:
* - detached/ref_child3/ (parent=fileid of the child, "detached" has no entry)
* - detached/ref_child3/child
*
* Normal files that should be untouched
* - files/untouched_folder
* - files/untouched.file
*/

// Test, corrupt storage
$storageId = 1;
$rootId1 = $this->createFileCacheEntry($storageId, '');
$baseId1 = $this->createFileCacheEntry($storageId, 'files', $rootId1);
Expand All @@ -308,11 +312,39 @@ public function testRepairSelfReferencing() {
// make it reference its own sub child
$this->setFileCacheEntryParent($refChild2Id, $refChild2ChildChildId);

$refChild3Id = $this->createFileCacheEntry($storageId, 'detached/ref_child3', $baseId1);
$willBeOverwritten = -1;
$refChild3Id = $this->createFileCacheEntry($storageId, 'detached/ref_child3', $willBeOverwritten);
$refChild3ChildId = $this->createFileCacheEntry($storageId, 'detached/ref_child3/child', $refChild3Id);
// make it reference its own child
$this->setFileCacheEntryParent($refChild3Id, $refChild3ChildId);

$untouchedFileId = $this->createFileCacheEntry($storageId, 'files/untouched.file', $baseId1);
$untouchedFolderId = $this->createFileCacheEntry($storageId, 'files/untouched_folder', $baseId1);
// End correct storage

// Parallel, correct, but identical storage - used to check for storage isolation and query scope
$storageId2 = 2;
$rootId2 = $this->createFileCacheEntry($storageId2, '');
$baseId2 = $this->createFileCacheEntry($storageId2, 'files', $rootId2);

$selfRefId_parallel = $this->createFileCacheEntry($storageId2, 'files/all_your_zombies', $baseId2);

$refChild1Id_parallel = $this->createFileCacheEntry($storageId2, 'files/ref_child1', $baseId2);
$refChild1ChildId_parallel = $this->createFileCacheEntry($storageId2, 'files/ref_child1/child', $refChild1Id_parallel);

$refChild2Id_parallel = $this->createFileCacheEntry($storageId2, 'files/ref_child2', $baseId2);
$refChild2ChildId_parallel = $this->createFileCacheEntry($storageId2, 'files/ref_child2/child', $refChild2Id_parallel);
$refChild2ChildChildId_parallel = $this->createFileCacheEntry($storageId2, 'files/ref_child2/child/child', $refChild2ChildId_parallel);

$refChild3DetachedId_parallel = $this->createFileCacheEntry($storageId2, 'detached', $rootId2);
$refChild3Id_parallel = $this->createFileCacheEntry($storageId2, 'detached/ref_child3', $refChild3DetachedId_parallel);
$refChild3ChildId_parallel = $this->createFileCacheEntry($storageId2, 'detached/ref_child3/child', $refChild3Id_parallel);

$untouchedFileId_parallel = $this->createFileCacheEntry($storageId2, 'files/untouched.file', $baseId2);
$untouchedFolderId_parallel = $this->createFileCacheEntry($storageId2, 'files/untouched_folder', $baseId2);
// End parallel storage


$outputMock = $this->createMock(IOutput::class);
$this->repair->setStorageNumericId($storageId);
$this->repair->run($outputMock);
Expand Down Expand Up @@ -386,8 +418,106 @@ public function testRepairSelfReferencing() {
$this->assertEquals((string)$storageId, $entry['storage']);
$this->assertEquals('detached', $entry['path']);
$this->assertEquals(md5('detached'), $entry['path_hash']);

// untouched file and folder are untouched
$entry = $this->getFileCacheEntry($untouchedFileId);
$this->assertEquals($baseId1, $entry['parent']);
$this->assertEquals((string)$storageId, $entry['storage']);
$this->assertEquals('files/untouched.file', $entry['path']);
$this->assertEquals(md5('files/untouched.file'), $entry['path_hash']);
$entry = $this->getFileCacheEntry($untouchedFolderId);
$this->assertEquals($baseId1, $entry['parent']);
$this->assertEquals((string)$storageId, $entry['storage']);
$this->assertEquals('files/untouched_folder', $entry['path']);
$this->assertEquals(md5('files/untouched_folder'), $entry['path_hash']);

// check that parallel storage is untouched
// self-referencing updated
$entry = $this->getFileCacheEntry($selfRefId_parallel);
$this->assertEquals($baseId2, $entry['parent']);
$this->assertEquals((string)$storageId2, $entry['storage']);
$this->assertEquals('files/all_your_zombies', $entry['path']);
$this->assertEquals(md5('files/all_your_zombies'), $entry['path_hash']);

// ref child 1 case was reparented to "files"
$entry = $this->getFileCacheEntry($refChild1Id_parallel);
$this->assertEquals($baseId2, $entry['parent']);
$this->assertEquals((string)$storageId2, $entry['storage']);
$this->assertEquals('files/ref_child1', $entry['path']);
$this->assertEquals(md5('files/ref_child1'), $entry['path_hash']);

// ref child 1 child left alone
$entry = $this->getFileCacheEntry($refChild1ChildId_parallel);
$this->assertEquals($refChild1Id_parallel, $entry['parent']);
$this->assertEquals((string)$storageId2, $entry['storage']);
$this->assertEquals('files/ref_child1/child', $entry['path']);
$this->assertEquals(md5('files/ref_child1/child'), $entry['path_hash']);

// ref child 2 case was reparented to "files"
$entry = $this->getFileCacheEntry($refChild2Id_parallel);
$this->assertEquals($baseId2, $entry['parent']);
$this->assertEquals((string)$storageId2, $entry['storage']);
$this->assertEquals('files/ref_child2', $entry['path']);
$this->assertEquals(md5('files/ref_child2'), $entry['path_hash']);

// ref child 2 child left alone
$entry = $this->getFileCacheEntry($refChild2ChildId_parallel);
$this->assertEquals($refChild2Id_parallel, $entry['parent']);
$this->assertEquals((string)$storageId2, $entry['storage']);
$this->assertEquals('files/ref_child2/child', $entry['path']);
$this->assertEquals(md5('files/ref_child2/child'), $entry['path_hash']);

// ref child 2 child child left alone
$entry = $this->getFileCacheEntry($refChild2ChildChildId_parallel);
$this->assertEquals($refChild2ChildId_parallel, $entry['parent']);
$this->assertEquals((string)$storageId2, $entry['storage']);
$this->assertEquals('files/ref_child2/child/child', $entry['path']);
$this->assertEquals(md5('files/ref_child2/child/child'), $entry['path_hash']);

// root entry left alone
$entry = $this->getFileCacheEntry($rootId2);
$this->assertEquals(-1, $entry['parent']);
$this->assertEquals((string)$storageId2, $entry['storage']);
$this->assertEquals('', $entry['path']);
$this->assertEquals(md5(''), $entry['path_hash']);

// ref child 3 child left alone
$entry = $this->getFileCacheEntry($refChild3ChildId_parallel);
$this->assertEquals($refChild3Id_parallel, $entry['parent']);
$this->assertEquals((string)$storageId2, $entry['storage']);
$this->assertEquals('detached/ref_child3/child', $entry['path']);
$this->assertEquals(md5('detached/ref_child3/child'), $entry['path_hash']);

// ref child 3 case was reparented to a new "detached" entry
$entry = $this->getFileCacheEntry($refChild3Id_parallel);
$this->assertTrue(isset($entry['parent']));
$this->assertEquals((string)$storageId2, $entry['storage']);
$this->assertEquals('detached/ref_child3', $entry['path']);
$this->assertEquals(md5('detached/ref_child3'), $entry['path_hash']);

// entry "detached" was untouched
$entry = $this->getFileCacheEntry($entry['parent']);
$this->assertEquals($rootId2, $entry['parent']);
$this->assertEquals((string)$storageId2, $entry['storage']);
$this->assertEquals('detached', $entry['path']);
$this->assertEquals(md5('detached'), $entry['path_hash']);

// untouched file and folder are untouched
$entry = $this->getFileCacheEntry($untouchedFileId_parallel);
$this->assertEquals($baseId2, $entry['parent']);
$this->assertEquals((string)$storageId2, $entry['storage']);
$this->assertEquals('files/untouched.file', $entry['path']);
$this->assertEquals(md5('files/untouched.file'), $entry['path_hash']);
$entry = $this->getFileCacheEntry($untouchedFolderId_parallel);
$this->assertEquals($baseId2, $entry['parent']);
$this->assertEquals((string)$storageId2, $entry['storage']);
$this->assertEquals('files/untouched_folder', $entry['path']);
$this->assertEquals(md5('files/untouched_folder'), $entry['path_hash']);

// end testing parallel storage
}


/**
* Test repair wrong parent id
*/
Expand Down Expand Up @@ -439,6 +569,8 @@ public function testRepairDetachedSubtree() {
* - files/missingdir/orphaned1 (orphaned entry as "missingdir" is missing)
* - missingdir/missingdir1/orphaned2 (orphaned entry two levels up to root)
*/

// Corrupt storage
$storageId = 1;
$rootId1 = $this->createFileCacheEntry($storageId, '');
$baseId1 = $this->createFileCacheEntry($storageId, 'files', $rootId1);
Expand All @@ -448,6 +580,18 @@ public function testRepairDetachedSubtree() {

$nonExistingParentId2 = $this->createNonExistingId();
$orphanedId2 = $this->createFileCacheEntry($storageId, 'missingdir/missingdir1/orphaned2', $nonExistingParentId2);
// end corrupt storage

// Parallel test storage
$storageId_parallel = 2;
$rootId1_parallel = $this->createFileCacheEntry($storageId_parallel, '');
$baseId1_parallel = $this->createFileCacheEntry($storageId_parallel, 'files', $rootId1_parallel);
$notOrphanedFolder_parallel = $this->createFileCacheEntry($storageId_parallel, 'files/missingdir', $baseId1_parallel);
$notOrphanedId1_parallel = $this->createFileCacheEntry($storageId_parallel, 'files/missingdir/orphaned1', $notOrphanedFolder_parallel);
$notOrphanedFolder2_parallel = $this->createFileCacheEntry($storageId_parallel, 'missingdir', $rootId1_parallel);
$notOrphanedFolderChild2_parallel = $this->createFileCacheEntry($storageId_parallel, 'missingdir/missingdir1', $notOrphanedFolder2_parallel);
$notOrphanedId2_parallel = $this->createFileCacheEntry($storageId_parallel, 'missingdir/missingdir1/orphaned2', $notOrphanedFolder2_parallel);
// end parallel test storage

$outputMock = $this->createMock(IOutput::class);
$this->repair->setStorageNumericId($storageId);
Expand Down Expand Up @@ -494,6 +638,49 @@ public function testRepairDetachedSubtree() {
$this->assertEquals((string)$storageId, $entry['storage']);
$this->assertEquals('', $entry['path']);
$this->assertEquals(md5(''), $entry['path_hash']);

// now check the parallel storage is intact
// orphaned entry reattached
$entry = $this->getFileCacheEntry($notOrphanedId1_parallel);
$this->assertEquals($notOrphanedFolder_parallel, $entry['parent']);
$this->assertEquals((string)$storageId_parallel, $entry['storage']);
$this->assertEquals('files/missingdir/orphaned1', $entry['path']);
$this->assertEquals(md5('files/missingdir/orphaned1'), $entry['path_hash']);

// not orphaned folder still exists
$entry = $this->getFileCacheEntry($notOrphanedFolder_parallel);
$this->assertEquals($baseId1_parallel, $entry['parent']);
$this->assertEquals((string)$storageId_parallel, $entry['storage']);
$this->assertEquals('files/missingdir', $entry['path']);
$this->assertEquals(md5('files/missingdir'), $entry['path_hash']);

// not orphaned entry still exits
$entry = $this->getFileCacheEntry($notOrphanedId2_parallel);
$this->assertEquals($notOrphanedFolder2_parallel, $entry['parent']);
$this->assertEquals((string)$storageId_parallel, $entry['storage']);
$this->assertEquals('missingdir/missingdir1/orphaned2', $entry['path']);
$this->assertEquals(md5('missingdir/missingdir1/orphaned2'), $entry['path_hash']);

// non existing id exists now
$entry = $this->getFileCacheEntry($notOrphanedFolderChild2_parallel);
$this->assertEquals($notOrphanedFolder2_parallel, $entry['parent']);
$this->assertEquals((string)$storageId_parallel, $entry['storage']);
$this->assertEquals('missingdir/missingdir1', $entry['path']);
$this->assertEquals(md5('missingdir/missingdir1'), $entry['path_hash']);

// non existing id parent exists now
$entry = $this->getFileCacheEntry($notOrphanedFolder2_parallel);
$this->assertEquals($rootId1_parallel, $entry['parent']);
$this->assertEquals((string)$storageId_parallel, $entry['storage']);
$this->assertEquals('missingdir', $entry['path']);
$this->assertEquals(md5('missingdir'), $entry['path_hash']);

// root entry left alone
$entry = $this->getFileCacheEntry($rootId1_parallel);
$this->assertEquals(-1, $entry['parent']);
$this->assertEquals((string)$storageId_parallel, $entry['storage']);
$this->assertEquals('', $entry['path']);
$this->assertEquals(md5(''), $entry['path_hash']);
}

/**
Expand All @@ -507,6 +694,12 @@ public function testRepairMissingRoot() {
$nonExistingParentId = $this->createNonExistingId();
$orphanedId = $this->createFileCacheEntry($storageId, 'noroot', $nonExistingParentId);

// Test parallel storage which should be untouched by the repair operation
$testStorageId = 2;
$baseId = $this->createFileCacheEntry($testStorageId, '');
$noRootid = $this->createFileCacheEntry($testStorageId, 'noroot', $baseId);


$outputMock = $this->createMock(IOutput::class);
$this->repair->setStorageNumericId($storageId);
$this->repair->run($outputMock);
Expand All @@ -524,6 +717,18 @@ public function testRepairMissingRoot() {
$this->assertEquals((string)$storageId, $entry['storage']);
$this->assertEquals('', $entry['path']);
$this->assertEquals(md5(''), $entry['path_hash']);

// Check that the parallel test storage is still intact
$entry = $this->getFileCacheEntry($noRootid);
$this->assertEquals($baseId, $entry['parent']);
$this->assertEquals((string)$testStorageId, $entry['storage']);
$this->assertEquals('noroot', $entry['path']);
$this->assertEquals(md5('noroot'), $entry['path_hash']);
$entry = $this->getFileCacheEntry($baseId);
$this->assertEquals(-1, $entry['parent']);
$this->assertEquals((string)$testStorageId, $entry['storage']);
$this->assertEquals('', $entry['path']);
$this->assertEquals(md5(''), $entry['path_hash']);
}
}