feat: Move mimetype repair to background jobs #39290
feat: Move mimetype repair to background jobs
Signed-off-by: Julius Härtl <[email protected]>
- Loading branch information
commit a617f1ef3437934265b74a4a97659e68fedf4368
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,257 @@ | ||
| <?php | ||
| /** | ||
| * @copyright Copyright (c) 2016, ownCloud, Inc. | ||
| * | ||
| * @author Arthur Schiwon <[email protected]> | ||
| * @author Christoph Wurst <[email protected]> | ||
| * @author Joas Schilling <[email protected]> | ||
| * @author Julius Härtl <[email protected]> | ||
| * @author Morris Jobke <[email protected]> | ||
| * @author nik gaffney <[email protected]> | ||
| * @author Olivier Paroz <[email protected]> | ||
| * @author Rello <[email protected]> | ||
| * @author Roeland Jago Douma <[email protected]> | ||
| * @author Stefan Weil <[email protected]> | ||
| * @author Thomas Ebert <[email protected]> | ||
| * @author Thomas Müller <[email protected]> | ||
| * @author Vincent Petry <[email protected]> | ||
| * | ||
| * @license AGPL-3.0 | ||
| * | ||
| * This code is free software: you can redistribute it and/or modify | ||
| * it under the terms of the GNU Affero General Public License, version 3, | ||
| * as published by the Free Software Foundation. | ||
| * | ||
| * This program is distributed in the hope that it will be useful, | ||
| * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| * GNU Affero General Public License for more details. | ||
| * | ||
| * You should have received a copy of the GNU Affero General Public License, version 3, | ||
| * along with this program. If not, see <http://www.gnu.org/licenses/> | ||
| * | ||
| */ | ||
| namespace OC\Repair; | ||
|
|
||
| use OCP\BackgroundJob\IJobList; | ||
| use OCP\IConfig; | ||
| use OCP\IDBConnection; | ||
| use OCP\Migration\IOutput; | ||
| use OCP\Migration\IRepairStep; | ||
|
|
||
| class AddRepairMimeTypeJob implements IRepairStep { | ||
| protected $config; | ||
| protected $connection; | ||
| protected $jobList; | ||
| public function __construct(IConfig $config, | ||
| IDBConnection $connection, IJobList $jobList) { | ||
| $this->config = $config; | ||
| $this->connection = $connection; | ||
| $this->jobList = $jobList; | ||
| } | ||
|
|
||
| public function getName() { | ||
| return 'Repair mime types through scheduling background jobs'; | ||
| } | ||
|
|
||
| private function scheduleMimeTypeUpdateJob(array $updatedMimetypes): int { | ||
| $qb = $this->connection->getQueryBuilder(); | ||
| $result = $qb | ||
| ->select('numeric_id') | ||
| ->from('storages') | ||
| ->execute(); | ||
|
|
||
| $jobCount = 0; | ||
| while ($row = $result->fetch()) { | ||
| $this->jobList->add(RepairMimeTypeJob::class, [ | ||
| 'storageId' => $row['numeric_id'], | ||
| 'mimetypes' => $updatedMimetypes, | ||
| ]); | ||
| $jobCount++; | ||
| } | ||
|
|
||
| return $jobCount; | ||
| } | ||
|
|
||
| private function introduceAsciidocType() { | ||
| $updatedMimetypes = [ | ||
| 'adoc' => 'text/asciidoc', | ||
| 'asciidoc' => 'text/asciidoc', | ||
| ]; | ||
|
|
||
| return $this->scheduleMimeTypeUpdateJob($updatedMimetypes); | ||
| } | ||
|
|
||
| private function introduceImageTypes() { | ||
| $updatedMimetypes = [ | ||
| 'jp2' => 'image/jp2', | ||
| 'webp' => 'image/webp', | ||
| ]; | ||
|
|
||
| return $this->scheduleMimeTypeUpdateJob($updatedMimetypes); | ||
| } | ||
|
|
||
| private function introduceWindowsProgramTypes() { | ||
| $updatedMimetypes = [ | ||
| 'htaccess' => 'text/plain', | ||
| 'bat' => 'application/x-msdos-program', | ||
| 'cmd' => 'application/cmd', | ||
| ]; | ||
|
|
||
| return $this->scheduleMimeTypeUpdateJob($updatedMimetypes); | ||
| } | ||
|
|
||
| private function introduceLocationTypes() { | ||
| $updatedMimetypes = [ | ||
| 'gpx' => 'application/gpx+xml', | ||
| 'kml' => 'application/vnd.google-earth.kml+xml', | ||
| 'kmz' => 'application/vnd.google-earth.kmz', | ||
| 'tcx' => 'application/vnd.garmin.tcx+xml', | ||
| ]; | ||
|
|
||
| return $this->scheduleMimeTypeUpdateJob($updatedMimetypes); | ||
| } | ||
|
|
||
| private function introduceInternetShortcutTypes() { | ||
| $updatedMimetypes = [ | ||
| 'url' => 'application/internet-shortcut', | ||
| 'webloc' => 'application/internet-shortcut' | ||
| ]; | ||
|
|
||
| return $this->scheduleMimeTypeUpdateJob($updatedMimetypes); | ||
| } | ||
|
|
||
| private function introduceStreamingTypes() { | ||
| $updatedMimetypes = [ | ||
| 'm3u' => 'audio/mpegurl', | ||
| 'm3u8' => 'audio/mpegurl', | ||
| 'pls' => 'audio/x-scpls' | ||
| ]; | ||
|
|
||
| return $this->scheduleMimeTypeUpdateJob($updatedMimetypes); | ||
| } | ||
|
|
||
| private function introduceVisioTypes() { | ||
| $updatedMimetypes = [ | ||
| 'vsdm' => 'application/vnd.visio', | ||
| 'vsdx' => 'application/vnd.visio', | ||
| 'vssm' => 'application/vnd.visio', | ||
| 'vssx' => 'application/vnd.visio', | ||
| 'vstm' => 'application/vnd.visio', | ||
| 'vstx' => 'application/vnd.visio', | ||
| ]; | ||
|
|
||
| return $this->scheduleMimeTypeUpdateJob($updatedMimetypes); | ||
| } | ||
|
|
||
| private function introduceComicbookTypes() { | ||
| $updatedMimetypes = [ | ||
| 'cb7' => 'application/comicbook+7z', | ||
| 'cba' => 'application/comicbook+ace', | ||
| 'cbr' => 'application/comicbook+rar', | ||
| 'cbt' => 'application/comicbook+tar', | ||
| 'cbtc' => 'application/comicbook+truecrypt', | ||
| 'cbz' => 'application/comicbook+zip', | ||
| ]; | ||
|
|
||
| return $this->scheduleMimeTypeUpdateJob($updatedMimetypes); | ||
| } | ||
|
|
||
| private function introduceOpenDocumentTemplates() { | ||
| $updatedMimetypes = [ | ||
| 'ott' => 'application/vnd.oasis.opendocument.text-template', | ||
| 'ots' => 'application/vnd.oasis.opendocument.spreadsheet-template', | ||
| 'otp' => 'application/vnd.oasis.opendocument.presentation-template', | ||
| 'otg' => 'application/vnd.oasis.opendocument.graphics-template', | ||
| ]; | ||
|
|
||
| return $this->scheduleMimeTypeUpdateJob($updatedMimetypes); | ||
| } | ||
|
|
||
| private function introduceFlatOpenDocumentType() { | ||
| $updatedMimetypes = [ | ||
| "fodt" => "application/vnd.oasis.opendocument.text-flat-xml", | ||
| "fods" => "application/vnd.oasis.opendocument.spreadsheet-flat-xml", | ||
| "fodg" => "application/vnd.oasis.opendocument.graphics-flat-xml", | ||
| "fodp" => "application/vnd.oasis.opendocument.presentation-flat-xml", | ||
| ]; | ||
|
|
||
| return $this->scheduleMimeTypeUpdateJob($updatedMimetypes); | ||
| } | ||
|
|
||
| private function introduceOrgModeType() { | ||
| $updatedMimetypes = [ | ||
| 'org' => 'text/org' | ||
| ]; | ||
|
|
||
| return $this->scheduleMimeTypeUpdateJob($updatedMimetypes); | ||
| } | ||
|
|
||
| private function introduceOnlyofficeFormType() { | ||
| $updatedMimetypes = [ | ||
| "oform" => "application/vnd.openxmlformats-officedocument.wordprocessingml.document.oform", | ||
| "docxf" => "application/vnd.openxmlformats-officedocument.wordprocessingml.document.docxf", | ||
| ]; | ||
|
|
||
| return $this->scheduleMimeTypeUpdateJob($updatedMimetypes); | ||
| } | ||
|
|
||
|
|
||
| /** | ||
| * Fix mime types | ||
| */ | ||
| public function run(IOutput $out) { | ||
| $ocVersionFromBeforeUpdate = $this->config->getSystemValueString('version', '0.0.0'); | ||
|
|
||
| // NOTE TO DEVELOPERS: when adding new mime types, please make sure to | ||
| // add a version comparison to avoid doing it every time | ||
|
|
||
| if (version_compare($ocVersionFromBeforeUpdate, '12.0.0.14', '<') && $this->introduceImageTypes()) { | ||
| $out->info('Fixed image mime types'); | ||
| } | ||
|
|
||
| if (version_compare($ocVersionFromBeforeUpdate, '12.0.0.13', '<') && $this->introduceWindowsProgramTypes()) { | ||
| $out->info('Fixed windows program mime types'); | ||
| } | ||
|
|
||
| if (version_compare($ocVersionFromBeforeUpdate, '13.0.0.0', '<') && $this->introduceLocationTypes()) { | ||
| $out->info('Fixed geospatial mime types'); | ||
| } | ||
|
|
||
| if (version_compare($ocVersionFromBeforeUpdate, '13.0.0.3', '<') && $this->introduceInternetShortcutTypes()) { | ||
| $out->info('Fixed internet-shortcut mime types'); | ||
| } | ||
|
|
||
| if (version_compare($ocVersionFromBeforeUpdate, '13.0.0.6', '<') && $this->introduceStreamingTypes()) { | ||
| $out->info('Fixed streaming mime types'); | ||
| } | ||
|
|
||
| if (version_compare($ocVersionFromBeforeUpdate, '14.0.0.8', '<') && $this->introduceVisioTypes()) { | ||
| $out->info('Fixed visio mime types'); | ||
| } | ||
|
|
||
| if (version_compare($ocVersionFromBeforeUpdate, '14.0.0.10', '<') && $this->introduceComicbookTypes()) { | ||
| $out->info('Fixed comicbook mime types'); | ||
| } | ||
|
|
||
| if (version_compare($ocVersionFromBeforeUpdate, '20.0.0.5', '<') && $this->introduceOpenDocumentTemplates()) { | ||
| $out->info('Fixed OpenDocument template mime types'); | ||
| } | ||
|
|
||
| if (version_compare($ocVersionFromBeforeUpdate, '21.0.0.7', '<') && $this->introduceOrgModeType()) { | ||
| $out->info('Fixed orgmode mime types'); | ||
| } | ||
|
|
||
| if (version_compare($ocVersionFromBeforeUpdate, '23.0.0.2', '<') && $this->introduceFlatOpenDocumentType()) { | ||
| $out->info('Fixed Flat OpenDocument mime types'); | ||
| } | ||
|
|
||
| if (version_compare($ocVersionFromBeforeUpdate, '25.0.0.2', '<') && $this->introduceOnlyofficeFormType()) { | ||
| $out->info('Fixed ONLYOFFICE Forms OpenXML mime types'); | ||
| } | ||
|
|
||
| if (version_compare($ocVersionFromBeforeUpdate, '26.0.0.1', '<') && $this->introduceAsciidocType()) { | ||
| $out->info('Fixed AsciiDoc mime types'); | ||
| } | ||
| } | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,75 @@ | ||
| <?php | ||
|
|
||
| namespace OC\Repair; | ||
|
|
||
| use OCP\AppFramework\Utility\ITimeFactory; | ||
| use OCP\BackgroundJob\QueuedJob; | ||
| use OCP\DB\QueryBuilder\IQueryBuilder; | ||
| use OCP\IDBConnection; | ||
| use Psr\Log\LoggerInterface; | ||
|
|
||
| class RepairMimeTypeJob extends QueuedJob { | ||
| private IDBConnection $connection; | ||
| private LoggerInterface $logger; | ||
|
|
||
| public function __construct(ITimeFactory $time, IDBConnection $connection, LoggerInterface $logger) { | ||
| parent::__construct($time); | ||
| $this->connection = $connection; | ||
| $this->logger = $logger; | ||
| } | ||
|
|
||
| protected function run($argument) { | ||
| $storageId = (int)$argument['storageId']; | ||
| $mimetypes = $argument['mimetypes']; | ||
| $this->updateMimetypesForStorage($storageId, $mimetypes); | ||
| } | ||
|
|
||
| private function updateMimetypesForStorage(int $storageId, array $updatedMimetypes) { | ||
| $query = $this->connection->getQueryBuilder(); | ||
| $query->select('id') | ||
| ->from('mimetypes') | ||
| ->where($query->expr()->eq('mimetype', $query->createParameter('mimetype'), IQueryBuilder::PARAM_INT)); | ||
| $insert = $this->connection->getQueryBuilder(); | ||
| $insert->insert('mimetypes') | ||
| ->setValue('mimetype', $insert->createParameter('mimetype')); | ||
|
|
||
| if (empty($this->folderMimeTypeId)) { | ||
| $query->setParameter('mimetype', 'httpd/unix-directory'); | ||
| $result = $query->execute(); | ||
| $this->folderMimeTypeId = (int)$result->fetchOne(); | ||
Check failureCode scanning / Psalm UndefinedThisPropertyAssignment |
||
| $result->closeCursor(); | ||
| } | ||
|
|
||
| $update = $this->connection->getQueryBuilder(); | ||
| $update->update('filecache') | ||
| ->set('mimetype', $update->createParameter('mimetype')) | ||
| ->where($update->expr()->eq('storage', $update->createParameter('storage'), IQueryBuilder::PARAM_INT)) | ||
| ->andWhere($update->expr()->neq('mimetype', $update->createParameter('mimetype'), IQueryBuilder::PARAM_INT)) | ||
| ->andWhere($update->expr()->neq('mimetype', $update->createParameter('folder'), IQueryBuilder::PARAM_INT)) | ||
| ->andWhere($update->expr()->iLike('name', $update->createParameter('name'))) | ||
| ->setParameter('folder', $this->folderMimeTypeId) | ||
| ->setParameter('storage', $storageId); | ||
|
|
||
| $count = 0; | ||
| foreach ($updatedMimetypes as $extension => $mimetype) { | ||
| // get target mimetype id | ||
| $query->setParameter('mimetype', $mimetype); | ||
| $result = $query->execute(); | ||
| $mimetypeId = (int)$result->fetchOne(); | ||
| $result->closeCursor(); | ||
|
|
||
| if (!$mimetypeId) { | ||
| // insert mimetype | ||
| $insert->setParameter('mimetype', $mimetype); | ||
| $insert->execute(); | ||
| $mimetypeId = $insert->getLastInsertId(); | ||
| } | ||
|
|
||
| // change mimetype for files with x extension | ||
| $update->setParameter('mimetype', $mimetypeId) | ||
| ->setParameter('name', '%' . $this->connection->escapeLikeParameter('.' . $extension)); | ||
| $count = $update->execute(); | ||
| $this->logger->info('Updated storage ' . $storageId . ' with ' . $count . ' file entry mimetypes for ' . $extension . ' to ' . $mimetype); | ||
| } | ||
| } | ||
| } | ||
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Check failure
Code scanning / Psalm
ParamNameMismatch