Skip to content

[stable9] Filecache repair step #29491

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
PVince81 merged 21 commits into from
Nov 8, 2017
Merged

[stable9] Filecache repair step #29491

Changes from 1 commit
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
266a288
Add repair step to repair mismatch filecache paths
Jun 28, 2017
0f2e1d0
Repair detached filecache entries through occ files:scan
Jul 26, 2017
ac9828a
Use DI and add more tests for storage filter
Aug 2, 2017
3a5d6ed
Run phase 2 of repair on gathered affected storages from phase 1
Aug 16, 2017
5077237
occ files:scan --all --repair repairs all storages at once
Aug 16, 2017
ce09606
Improve output when repairing. Fix non-repair file:scan
tomneedham Aug 18, 2017
ef59387
Add parallel non-corrupt storages to filecache repair step to check f…
tomneedham Sep 12, 2017
0f9265b
Test without parallel storages
tomneedham Sep 14, 2017
93342d3
Fix lastinsertid for postgres wehn reusing fileid
tomneedham Sep 14, 2017
ed699d2
Try oracle tests without throwing exception for stray transactions
tomneedham Sep 15, 2017
a08d233
Make filecache repair step work with oc9
tomneedham Sep 18, 2017
14b1beb
fix casing of new repair step
tomneedham Sep 18, 2017
6a7ada1
Oracle hates empty strings
Sep 18, 2017
dc5ff81
Cleanup testcase and comments
tomneedham Sep 20, 2017
dec4b10
Fix path casing
tomneedham Sep 29, 2017
d681977
Dont use create mock
tomneedham Sep 29, 2017
30eadce
Catch output advance method
tomneedham Sep 29, 2017
11cde16
Make this work with php5.4
tomneedham Sep 29, 2017
808778a
Workaround for Oracle trigger for fileid
Sep 19, 2017
64453fc
Remove hardcoded logging
tomneedham Nov 8, 2017
002417a
Fix repair step output handling
Nov 8, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Remove hardcoded logging
  • Loading branch information
@tomneedham
tomneedham committed Nov 8, 2017
commit 64453fcc504e4ec3917de72e302bc16cb9b8c2e4
75 changes: 12 additions & 63 deletions lib/private/repair/repairmismatchfilecachepath.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ class RepairMismatchFileCachePath extends BasicEmitter implements \OC\RepairStep

/**
* @param \OCP\IDBConnection $connection
* @param IMimeTypeLoader $mimeLoader
*/
public function __construct(IDBConnection $connection, IMimeTypeLoader $mimeLoader) {
$this->connection = $connection;
Expand Down Expand Up @@ -90,14 +91,13 @@ public function setCountOnly($countOnly) {
/**
* Fixes the broken entry's path.
*
* @param LogOutput $out repair output
* @param int $fileId file id of the entry to fix
* @param string $wrongPath wrong path of the entry to fix
* @param int $correctStorageNumericId numeric idea of the correct storage
* @param string $correctPath value to which to set the path of the entry
* @return bool true for success
*/
private function fixEntryPath(LogOutput $out, $fileId, $wrongPath, $correctStorageNumericId, $correctPath) {
private function fixEntryPath($fileId, $wrongPath, $correctStorageNumericId, $correctPath) {
// delete target if exists
$qb = $this->connection->getQueryBuilder();
$qb->delete('filecache')
Expand All @@ -122,7 +122,6 @@ private function fixEntryPath(LogOutput $out, $fileId, $wrongPath, $correctStora
if ($entryExisted) {
$text = " (replaced an existing entry)";
}
$out->advance(1, $text);
}

private function addQueryConditionsParentIdWrongPath($qb) {
Expand Down Expand Up @@ -217,7 +216,7 @@ private function countResultsToProcessNonExistingParentIdEntry($storageNumericId
/**
* Outputs a report about storages with wrong path that need repairing in the file cache
*/
private function reportAffectedStoragesParentIdWrongPath(LogOutput $out) {
private function reportAffectedStoragesParentIdWrongPath() {
$qb = $this->connection->getQueryBuilder();
$qb->selectDistinct('fc.storage');
$this->addQueryConditionsParentIdWrongPath($qb);
Expand All @@ -234,18 +233,12 @@ private function reportAffectedStoragesParentIdWrongPath(LogOutput $out) {
$storageIds[] = $row['storage'];
Copy link
Contributor Author

@PVince81 PVince81 Nov 8, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

you killed too much, the upgrade is not reporting any more.

We need to emit an event on $this object as it's a BasicEmmiter, it will output the entry on the console

}

if (!empty($storageIds)) {
$out->warning('The file cache contains entries with invalid path values for the following storage numeric ids: ' . implode(' ', $storageIds));
$out->warning('Please run `occ files:scan --all --repair` to repair'
.'all affected storages or run `occ files:scan userid --repair for '
.'each user with affected storages');
}
}

/**
* Outputs a report about storages with non existing parents that need repairing in the file cache
*/
private function reportAffectedStoragesNonExistingParentIdEntry(LogOutput $out) {
private function reportAffectedStoragesNonExistingParentIdEntry() {
$qb = $this->connection->getQueryBuilder();
$qb->selectDistinct('fc.storage');
$this->addQueryConditionsNonExistingParentIdEntry($qb);
Expand All @@ -262,21 +255,16 @@ private function reportAffectedStoragesNonExistingParentIdEntry(LogOutput $out)
$storageIds[] = $row['storage'];
}

if (!empty($storageIds)) {
$out->warning('The file cache contains entries where the parent id does not point to any existing entry for the following storage numeric ids: ' . implode(' ', $storageIds));
$out->warning('Please run `occ files:scan --all --repair` to repair all affected storages');
}
}

/**
* Repair all entries for which the parent entry exists but the path
* value doesn't match the parent's path.
*
* @param LogOutput $out
* @param int|null $storageNumericId storage to fix or null for all
* @return int[] storage numeric ids that were targets to a move and needs further fixing
*/
private function fixEntriesWithCorrectParentIdButWrongPath(LogOutput $out, $storageNumericId = null) {
private function fixEntriesWithCorrectParentIdButWrongPath($storageNumericId = null) {
$totalResultsCount = 0;
$affectedStorages = [$storageNumericId => true];

Expand Down Expand Up @@ -312,7 +300,6 @@ private function fixEntriesWithCorrectParentIdButWrongPath(LogOutput $out, $stor
// here because the reason we reached this code is because we already
// found it
$this->fixEntryParent(
$out,
$row['storage'],
$row['fileid'],
$row['path'],
Expand All @@ -321,7 +308,6 @@ private function fixEntriesWithCorrectParentIdButWrongPath(LogOutput $out, $stor
);
} else {
$this->fixEntryPath(
$out,
$row['fileid'],
$wrongPath,
$row['parentstorage'],
Expand All @@ -340,10 +326,6 @@ private function fixEntriesWithCorrectParentIdButWrongPath(LogOutput $out, $stor
// until all possible entries were fixed
} while ($lastResultsCount > 0);

if ($totalResultsCount > 0) {
$out->info("Fixed $totalResultsCount file cache entries with wrong path");
}

return array_keys($affectedStorages);
}

Expand Down Expand Up @@ -435,7 +417,6 @@ private function getOrCreateEntry($storageId, $path, $reuseFileId = null) {
/**
* Fixes the broken entry's path.
*
* @param LogOutput $out repair output
* @param int $storageId storage id of the entry to fix
* @param int $fileId file id of the entry to fix
* @param string $path path from the entry to fix
Expand All @@ -444,7 +425,7 @@ private function getOrCreateEntry($storageId, $path, $reuseFileId = null) {
* false if it doesn't
* @return bool true if the entry was fixed, false otherwise
*/
private function fixEntryParent(LogOutput $out, $storageId, $fileId, $path, $wrongParentId, $parentIdExists = false) {
private function fixEntryParent($storageId, $fileId, $path, $wrongParentId, $parentIdExists = false) {
if (!$parentIdExists) {
// if the parent doesn't exist, let us reuse its id in case there is metadata to salvage
$correctParentId = $this->getOrCreateEntry($storageId, dirname($path), $wrongParentId);
Expand All @@ -461,9 +442,6 @@ private function fixEntryParent(LogOutput $out, $storageId, $fileId, $path, $wro
->where($qb->expr()->eq('fileid', $qb->createNamedParameter($fileId)));
$qb->execute();

$text = "Fixed file cache entry with fileid $fileId, set wrong parent \"$wrongParentId\" to \"$correctParentId\"";
$out->advance(1, $text);

$this->connection->commit();

return true;
Expand All @@ -473,11 +451,10 @@ private function fixEntryParent(LogOutput $out, $storageId, $fileId, $path, $wro
* Repair entries where the parent id doesn't point to any existing entry
* by finding the actual parent entry matching the entry's path dirname.
*
* @param LogOutput $out output
* @param int|null $storageNumericId storage to fix or null for all
* @return int number of results that were fixed
*/
private function fixEntriesWithNonExistingParentIdEntry(LogOutput $out, $storageNumericId = null) {
private function fixEntriesWithNonExistingParentIdEntry($storageNumericId = null) {
$qb = $this->connection->getQueryBuilder();
$this->addQueryConditionsNonExistingParentIdEntry($qb, $storageNumericId);
$qb->setMaxResults(self::CHUNK_SIZE);
Expand All @@ -493,7 +470,6 @@ private function fixEntriesWithNonExistingParentIdEntry(LogOutput $out, $storage
$lastResultsCount = 0;
foreach ($rows as $row) {
$this->fixEntryParent(
$out,
$row['storage'],
$row['fileid'],
$row['path'],
Expand All @@ -511,26 +487,20 @@ private function fixEntriesWithNonExistingParentIdEntry(LogOutput $out, $storage
// until all possible entries were fixed
} while ($lastResultsCount > 0);

if ($totalResultsCount > 0) {
$out->info("Fixed $totalResultsCount file cache entries with wrong path");
}

return $totalResultsCount;
}

/**
* Run the repair step
*/
public function run() {

$out = new LogOutput(\OC::$server->getLogger());

$this->dirMimeTypeId = $this->mimeLoader->getId('httpd/unix-directory');
$this->dirMimePartId = $this->mimeLoader->getId('httpd');

if ($this->countOnly) {
$this->reportAffectedStoragesParentIdWrongPath($out);
$this->reportAffectedStoragesNonExistingParentIdEntry($out);
$this->reportAffectedStoragesParentIdWrongPath();
$this->reportAffectedStoragesNonExistingParentIdEntry();
} else {
$brokenPathEntries = $this->countResultsToProcessParentIdWrongPath($this->storageNumericId);
$brokenParentIdEntries = $this->countResultsToProcessNonExistingParentIdEntry($this->storageNumericId);
Expand All @@ -544,37 +514,16 @@ public function run() {
* This needs to be repaired by fixEntriesWithNonExistingParentIdEntry(), this is why
* we need to keep this specific order of repair.
*/
$affectedStorages = $this->fixEntriesWithCorrectParentIdButWrongPath($out, $this->storageNumericId);
$affectedStorages = $this->fixEntriesWithCorrectParentIdButWrongPath($this->storageNumericId);

if ($this->storageNumericId !== null) {
foreach ($affectedStorages as $storageNumericId) {
$this->fixEntriesWithNonExistingParentIdEntry($out, $storageNumericId);
$this->fixEntriesWithNonExistingParentIdEntry($storageNumericId);
}
} else {
// just fix all
$this->fixEntriesWithNonExistingParentIdEntry($out);
$this->fixEntriesWithNonExistingParentIdEntry();
}
$out->finishProgress();
$out->info('');
}
}
}

class LogOutput {
/**
* @var ILogger
*/
protected $logger;
public function __construct(ILogger $logger) {
$this->logger = $logger;
}
public function info($message) {
$this->logger->info($message);
}
public function finishProgress() {}
public function advance() {}
public function startProgress() {}
public function warning($message) {
$this->logger->warning($message);
}
}