Skip to content
Closed
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Added Delete class
Signed-off-by: charleypaulus <[email protected]>
  • Loading branch information
charleypaulus committed Nov 3, 2023
commit ee9c8e952f3e83eb282fe870824cffff1afd7e84
174 changes: 174 additions & 0 deletions core/Command/Preview/Delete.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,174 @@
<?php

declare(strict_types=1);

/**
* @copyright Copyright (c) 2021, Charley Paulus <[email protected]>
*
* @author Charley Paulus <[email protected]>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OC\Core\Command\Preview;

use OC\Preview\Storage\Root;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\Files\IMimeTypeLoader;
use OCP\Files\NotFoundException;
use OCP\Files\NotPermittedException;
use OCP\IDBConnection;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Output\OutputInterface;

class Delete extends Command {
protected IDBConnection $connection;
private Root $previewFolder;
private IMimeTypeLoader $mimeTypeLoader;

public function __construct(IDBConnection $connection,
Root $previewFolder,
IMimeTypeLoader $mimeTypeLoader) {
parent::__construct();

$this->connection = $connection;
$this->previewFolder = $previewFolder;
$this->mimeTypeLoader = $mimeTypeLoader;
}

protected function configure() {

Check notice

Code scanning / Psalm

MissingReturnType

Method OC\Core\Command\Preview\Delete::configure does not have a return type, expecting void
$this
->setName('preview:delete')
->setDescription('Deletes all previews')
->addOption('old-only', 'o', InputOption::VALUE_NONE, 'Limit deletion to old previews (no longer having their original file)')
->addOption('mimetype', 'm', InputArgument::OPTIONAL, 'Limit deletion to this mimetype, eg. --mimetype="image/jpeg"')
->addOption('dry', 'd', InputOption::VALUE_NONE, 'Dry mode (will not delete any files). In combination with the verbose mode one could check the operations');
}

protected function execute(InputInterface $input, OutputInterface $output): int {
$oldOnly = $input->getOption('old-only');

$selectedMimetype = $input->getOption('mimetype');
if ($selectedMimetype) {
if ($oldOnly) {
$output->writeln('Mimetype of absent original files cannot be determined. Aborting...');
return 0;
} else {
if (! $this->mimeTypeLoader->exists($selectedMimetype)) {
$output->writeln('Mimetype ' . $selectedMimetype . ' does not exist in database. Aborting...');
return 0;
}
}
}

$dryMode = $input->getOption('dry');
if ($dryMode) {
$output->writeln('INFO: The command is run in dry mode and will not modify anything.');
$output->writeln('');
}

$this->deletePreviews($output, $oldOnly, $selectedMimetype, $dryMode);

return 0;
}

private function deletePreviews(OutputInterface $output, bool $oldOnly, string $selectedMimetype = null, bool $dryMode): void {
$previewFoldersToDeleteCount = 0;

foreach ($this->getPreviewsToDelete($output, $oldOnly, $selectedMimetype) as ['name' => $previewFileId, 'path' => $filePath]) {
if ($oldOnly || $filePath === null) {
$output->writeln('Deleting previews of absent original file (fileid:' . $previewFileId . ')', OutputInterface::VERBOSITY_VERBOSE);
} else {
$output->writeln('Deleting previews of original file ' . substr($filePath, 7) . ' (fileid:' . $previewFileId . ')', OutputInterface::VERBOSITY_VERBOSE);
}

$previewFoldersToDeleteCount++;

if ($dryMode) {
continue;
}

try {
$preview = $this->previewFolder->getFolder((string)$previewFileId);
$preview->delete();
} catch (NotFoundException $e) {
// continue
} catch (NotPermittedException $e) {
// continue
}
}

$output->writeln('Deleted previews of ' . $previewFoldersToDeleteCount . ' original files');
}

// Copy pasted and adjusted from
// "lib/private/Preview/BackgroundCleanupJob.php".
private function getPreviewsToDelete(OutputInterface $output, bool $oldOnly, string $selectedMimetype = null): \Iterator {
// Get preview folder
$qb = $this->connection->getQueryBuilder();
$qb->select('path', 'mimetype')
->from('filecache')
->where($qb->expr()->eq('fileid', $qb->createNamedParameter($this->previewFolder->getId())));
$cursor = $qb->execute();
$data = $cursor->fetch();
$cursor->closeCursor();

$output->writeln('Preview folder: ' . $data['path'], OutputInterface::VERBOSITY_VERBOSE);

if ($data === null) {
return [];
}

// Get previews to delete
// Initialize Query Builder
$qb = $this->connection->getQueryBuilder();

/* This lovely like is the result of the way the new previews are stored
* We take the md5 of the name (fileid) and split the first 7 chars. That way
* there are not a gazillion files in the root of the preview appdata.*/
$like = $this->connection->escapeLikeParameter($data['path']) . '/_/_/_/_/_/_/_/%';

// Specify conditions based on options
$and = $qb->expr()->andX();
$and->add($qb->expr()->like('a.path', $qb->createNamedParameter($like)));
$and->add($qb->expr()->eq('a.mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('httpd/unix-directory'))));
if ($oldOnly) {
$and->add($qb->expr()->isNull('b.fileid'));
}
if ($selectedMimetype) {
$and->add($qb->expr()->eq('b.mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('image/jpeg'))));
}

// Build query
$qb->select('a.name', 'b.path')
->from('filecache', 'a')
->leftJoin('a', 'filecache', 'b', $qb->expr()->eq(
$qb->expr()->castColumn('a.name', IQueryBuilder::PARAM_INT), 'b.fileid'
))
->where($and);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Might still be a good idea to limit the results and rather run multiple queries after each other in a php loop to avoid massive queries against the db.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I can first narrow down the preview search on the storage that holds the preview folder.

Then do you have in mind something like "SELECT ... FROM ... LIMIT a OFFSET a*x", with for loop on x? If so, do you know what a good batch size would be?

If the issue is not the mysql query size itself, but on the call for deletions, these deletions are actually done in a for loop in deletePreviews function (line 109).

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I was thinking about the query size itself. You wouldn't need the offset as when going through the chunk the entries would be deleted from the table so the next query with the same limit should just give you the next batch.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I can first narrow down the preview search on the storage that holds the preview folder.

This should still be fine to keep in the query as the storage column is indexed.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I can first narrow down the preview search on the storage that holds the preview folder.

This should still be fine to keep in the query as the storage column is indexed.

I updated the sql query with a condition on storage.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I was thinking about the query size itself. You wouldn't need the offset as when going through the chunk the entries would be deleted from the table so the next query with the same limit should just give you the next batch.

I added an option to delete previews by batch.
Two notes:

  • Batch deletion is incompatible with dry mode, as it relies on actually deleting previews (the proposed code takes care of catching these incompatible options).
  • This option makes the code not in line with lib/private/Preview/BackgroundCleanupJob.php or core/Command/Preview/ResetRenderedTexts.php that do not have a batch mode.


$cursor = $qb->execute();

Check notice

Code scanning / Psalm

DeprecatedMethod

The method OCP\DB\QueryBuilder\IQueryBuilder::execute has been marked as deprecated

while ($row = $cursor->fetch()) {

Check notice

Code scanning / Psalm

PossiblyInvalidMethodCall

Cannot call method on possible int variable $cursor
yield $row;
}

$cursor->closeCursor();

Check notice

Code scanning / Psalm

PossiblyInvalidMethodCall

Cannot call method on possible int variable $cursor
}
}