From c3bc362f487cbea1458206e2b285210b38c0ea1e Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Thu, 20 Mar 2025 15:22:27 +0100 Subject: [PATCH 1/5] feat: add command to get object metadata Signed-off-by: Robin Appelman --- apps/files/appinfo/info.xml | 1 + .../composer/composer/autoload_classmap.php | 1 + .../composer/composer/autoload_static.php | 1 + apps/files/lib/Command/Object/Info.php | 79 +++++++++++++++++++ lib/composer/composer/autoload_classmap.php | 1 + lib/composer/composer/autoload_static.php | 1 + lib/private/Files/ObjectStore/S3.php | 42 +++++++++- .../ObjectStore/IObjectStoreMetaData.php | 36 +++++++++ 8 files changed, 159 insertions(+), 3 deletions(-) create mode 100644 apps/files/lib/Command/Object/Info.php create mode 100644 lib/public/Files/ObjectStore/IObjectStoreMetaData.php diff --git a/apps/files/appinfo/info.xml b/apps/files/appinfo/info.xml index cdcc79e47fb76..add3da17d548b 100644 --- a/apps/files/appinfo/info.xml +++ b/apps/files/appinfo/info.xml @@ -49,6 +49,7 @@ OCA\Files\Command\Object\Delete OCA\Files\Command\Object\Get OCA\Files\Command\Object\Put + OCA\Files\Command\Object\Info diff --git a/apps/files/composer/composer/autoload_classmap.php b/apps/files/composer/composer/autoload_classmap.php index 0d9e6aa2d77dd..0afe0261292af 100644 --- a/apps/files/composer/composer/autoload_classmap.php +++ b/apps/files/composer/composer/autoload_classmap.php @@ -35,6 +35,7 @@ 'OCA\\Files\\Command\\Move' => $baseDir . '/../lib/Command/Move.php', 'OCA\\Files\\Command\\Object\\Delete' => $baseDir . '/../lib/Command/Object/Delete.php', 'OCA\\Files\\Command\\Object\\Get' => $baseDir . '/../lib/Command/Object/Get.php', + 'OCA\\Files\\Command\\Object\\Info' => $baseDir . '/../lib/Command/Object/Info.php', 'OCA\\Files\\Command\\Object\\ObjectUtil' => $baseDir . '/../lib/Command/Object/ObjectUtil.php', 'OCA\\Files\\Command\\Object\\Put' => $baseDir . '/../lib/Command/Object/Put.php', 'OCA\\Files\\Command\\Put' => $baseDir . '/../lib/Command/Put.php', diff --git a/apps/files/composer/composer/autoload_static.php b/apps/files/composer/composer/autoload_static.php index 5ece90731783b..eab4a88b4ee29 100644 --- a/apps/files/composer/composer/autoload_static.php +++ b/apps/files/composer/composer/autoload_static.php @@ -50,6 +50,7 @@ class ComposerStaticInitFiles 'OCA\\Files\\Command\\Move' => __DIR__ . '/..' . '/../lib/Command/Move.php', 'OCA\\Files\\Command\\Object\\Delete' => __DIR__ . '/..' . '/../lib/Command/Object/Delete.php', 'OCA\\Files\\Command\\Object\\Get' => __DIR__ . '/..' . '/../lib/Command/Object/Get.php', + 'OCA\\Files\\Command\\Object\\Info' => __DIR__ . '/..' . '/../lib/Command/Object/Info.php', 'OCA\\Files\\Command\\Object\\ObjectUtil' => __DIR__ . '/..' . '/../lib/Command/Object/ObjectUtil.php', 'OCA\\Files\\Command\\Object\\Put' => __DIR__ . '/..' . '/../lib/Command/Object/Put.php', 'OCA\\Files\\Command\\Put' => __DIR__ . '/..' . '/../lib/Command/Put.php', diff --git a/apps/files/lib/Command/Object/Info.php b/apps/files/lib/Command/Object/Info.php new file mode 100644 index 0000000000000..5dcc552ea34d0 --- /dev/null +++ b/apps/files/lib/Command/Object/Info.php @@ -0,0 +1,79 @@ +setName('files:object:info') + ->setDescription('Get the metadata of an object') + ->addArgument('object', InputArgument::REQUIRED, 'Object to get') + ->addOption('bucket', 'b', InputOption::VALUE_REQUIRED, "Bucket to get the object from, only required in cases where it can't be determined from the config"); + } + + public function execute(InputInterface $input, OutputInterface $output): int { + $object = $input->getArgument('object'); + $objectStore = $this->objectUtils->getObjectStore($input->getOption('bucket'), $output); + if (!$objectStore) { + return self::FAILURE; + } + + if (!$objectStore instanceof IObjectStoreMetaData) { + $output->writeln('Configured object store does currently not support retrieve metadata'); + return self::FAILURE; + } + + if (!$objectStore->objectExists($object)) { + $output->writeln("Object $object does not exist"); + return self::FAILURE; + } + + try { + $meta = $objectStore->getObjectMetaData($object); + } catch (\Exception $e) { + $msg = $e->getMessage(); + $output->writeln("Failed to read $object from object store: $msg"); + return self::FAILURE; + } + + if ($input->getOption('output') === 'plain' && isset($meta['size'])) { + $meta['size'] = \OC_Helper::humanFileSize($meta['size']); + } + if (isset($meta['mtime'])) { + $meta['mtime'] = $meta['mtime']->format(\DateTimeImmutable::ATOM); + } + if (!isset($meta['mimetype'])) { + $handle = $objectStore->readObject($object); + $head = fread($handle, 8192); + fclose($handle); + $meta['mimetype'] = $this->mimeTypeDetector->detectString($head); + } + + $this->writeArrayInOutputFormat($input, $output, $meta); + + return self::SUCCESS; + } + +} diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index 2d0b62d749f7f..52e3075e4134b 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -456,6 +456,7 @@ 'OCP\\Files\\Notify\\INotifyHandler' => $baseDir . '/lib/public/Files/Notify/INotifyHandler.php', 'OCP\\Files\\Notify\\IRenameChange' => $baseDir . '/lib/public/Files/Notify/IRenameChange.php', 'OCP\\Files\\ObjectStore\\IObjectStore' => $baseDir . '/lib/public/Files/ObjectStore/IObjectStore.php', + 'OCP\\Files\\ObjectStore\\IObjectStoreMetaData' => $baseDir . '/lib/public/Files/ObjectStore/IObjectStoreMetaData.php', 'OCP\\Files\\ObjectStore\\IObjectStoreMultiPartUpload' => $baseDir . '/lib/public/Files/ObjectStore/IObjectStoreMultiPartUpload.php', 'OCP\\Files\\ReservedWordException' => $baseDir . '/lib/public/Files/ReservedWordException.php', 'OCP\\Files\\Search\\ISearchBinaryOperator' => $baseDir . '/lib/public/Files/Search/ISearchBinaryOperator.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index 64210bd584344..e98bc3e1aaaee 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -505,6 +505,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OCP\\Files\\Notify\\INotifyHandler' => __DIR__ . '/../../..' . '/lib/public/Files/Notify/INotifyHandler.php', 'OCP\\Files\\Notify\\IRenameChange' => __DIR__ . '/../../..' . '/lib/public/Files/Notify/IRenameChange.php', 'OCP\\Files\\ObjectStore\\IObjectStore' => __DIR__ . '/../../..' . '/lib/public/Files/ObjectStore/IObjectStore.php', + 'OCP\\Files\\ObjectStore\\IObjectStoreMetaData' => __DIR__ . '/../../..' . '/lib/public/Files/ObjectStore/IObjectStoreMetaData.php', 'OCP\\Files\\ObjectStore\\IObjectStoreMultiPartUpload' => __DIR__ . '/../../..' . '/lib/public/Files/ObjectStore/IObjectStoreMultiPartUpload.php', 'OCP\\Files\\ReservedWordException' => __DIR__ . '/../../..' . '/lib/public/Files/ReservedWordException.php', 'OCP\\Files\\Search\\ISearchBinaryOperator' => __DIR__ . '/../../..' . '/lib/public/Files/Search/ISearchBinaryOperator.php', diff --git a/lib/private/Files/ObjectStore/S3.php b/lib/private/Files/ObjectStore/S3.php index 41ab75caf45fe..63c93ca8f3e8f 100644 --- a/lib/private/Files/ObjectStore/S3.php +++ b/lib/private/Files/ObjectStore/S3.php @@ -3,14 +3,16 @@ * SPDX-FileCopyrightText: 2016 Nextcloud GmbH and Nextcloud contributors * SPDX-License-Identifier: AGPL-3.0-or-later */ + namespace OC\Files\ObjectStore; use Aws\Result; use Exception; use OCP\Files\ObjectStore\IObjectStore; +use OCP\Files\ObjectStore\IObjectStoreMetaData; use OCP\Files\ObjectStore\IObjectStoreMultiPartUpload; -class S3 implements IObjectStore, IObjectStoreMultiPartUpload { +class S3 implements IObjectStore, IObjectStoreMultiPartUpload, IObjectStoreMetaData { use S3ConnectionTrait; use S3ObjectTrait; @@ -61,7 +63,7 @@ public function getMultipartUploads(string $urn, string $uploadId): array { 'Key' => $urn, 'UploadId' => $uploadId, 'MaxParts' => 1000, - 'PartNumberMarker' => $partNumberMarker + 'PartNumberMarker' => $partNumberMarker, ] + $this->getSSECParameters()); $parts = array_merge($parts, $result->get('Parts') ?? []); $isTruncated = $result->get('IsTruncated'); @@ -89,7 +91,41 @@ public function abortMultipartUpload($urn, $uploadId): void { $this->getConnection()->abortMultipartUpload([ 'Bucket' => $this->bucket, 'Key' => $urn, - 'UploadId' => $uploadId + 'UploadId' => $uploadId, ]); } + + public function getObjectMetaData(string $urn): array { + $object = $this->getConnection()->headObject([ + 'Bucket' => $this->bucket, + 'Key' => $urn + ] + $this->getSSECParameters())->toArray(); + return [ + 'mtime' => $object['LastModified'], + 'etag' => trim($object['ETag'], '"'), + 'size' => (int)($object['Size'] ?? $object['ContentLength']), + ]; + } + + public function listObjects(string $prefix = ''): \Iterator { + $results = $this->getConnection()->getPaginator('ListObjectsV2', [ + 'Bucket' => $this->bucket, + 'Prefix' => $prefix, + ] + $this->getSSECParameters()); + + foreach ($results as $result) { + if (is_array($result['Contents'])) { + foreach ($result['Contents'] as $object) { + yield [ + 'urn' => basename($object['Key']), + 'meta' => [ + 'mtime' => strtotime($object['LastModified']), + 'etag' => trim($object['ETag'], '"'), + 'size' => (int)($object['Size'] ?? $object['ContentLength']), + ], + ]; + } + } + } + } } diff --git a/lib/public/Files/ObjectStore/IObjectStoreMetaData.php b/lib/public/Files/ObjectStore/IObjectStoreMetaData.php new file mode 100644 index 0000000000000..48e1290850ca7 --- /dev/null +++ b/lib/public/Files/ObjectStore/IObjectStoreMetaData.php @@ -0,0 +1,36 @@ + + */ + public function listObjects(string $prefix = ''): \Iterator; +} From fcde776683ad5c844e82770e66ef814105d6a7c8 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Thu, 20 Mar 2025 15:59:25 +0100 Subject: [PATCH 2/5] feat: add command to list objects Signed-off-by: Robin Appelman --- apps/files/appinfo/info.xml | 1 + .../composer/composer/autoload_classmap.php | 1 + .../composer/composer/autoload_static.php | 1 + apps/files/lib/Command/Object/ListObject.php | 119 ++++++++++++++++++ lib/private/Files/ObjectStore/S3.php | 4 +- .../ObjectStore/IObjectStoreMetaData.php | 4 +- 6 files changed, 127 insertions(+), 3 deletions(-) create mode 100644 apps/files/lib/Command/Object/ListObject.php diff --git a/apps/files/appinfo/info.xml b/apps/files/appinfo/info.xml index add3da17d548b..5f6f7e52c306e 100644 --- a/apps/files/appinfo/info.xml +++ b/apps/files/appinfo/info.xml @@ -50,6 +50,7 @@ OCA\Files\Command\Object\Get OCA\Files\Command\Object\Put OCA\Files\Command\Object\Info + OCA\Files\Command\Object\ListObject diff --git a/apps/files/composer/composer/autoload_classmap.php b/apps/files/composer/composer/autoload_classmap.php index 0afe0261292af..6489b69179839 100644 --- a/apps/files/composer/composer/autoload_classmap.php +++ b/apps/files/composer/composer/autoload_classmap.php @@ -36,6 +36,7 @@ 'OCA\\Files\\Command\\Object\\Delete' => $baseDir . '/../lib/Command/Object/Delete.php', 'OCA\\Files\\Command\\Object\\Get' => $baseDir . '/../lib/Command/Object/Get.php', 'OCA\\Files\\Command\\Object\\Info' => $baseDir . '/../lib/Command/Object/Info.php', + 'OCA\\Files\\Command\\Object\\ListObject' => $baseDir . '/../lib/Command/Object/ListObject.php', 'OCA\\Files\\Command\\Object\\ObjectUtil' => $baseDir . '/../lib/Command/Object/ObjectUtil.php', 'OCA\\Files\\Command\\Object\\Put' => $baseDir . '/../lib/Command/Object/Put.php', 'OCA\\Files\\Command\\Put' => $baseDir . '/../lib/Command/Put.php', diff --git a/apps/files/composer/composer/autoload_static.php b/apps/files/composer/composer/autoload_static.php index eab4a88b4ee29..dadac010aea63 100644 --- a/apps/files/composer/composer/autoload_static.php +++ b/apps/files/composer/composer/autoload_static.php @@ -51,6 +51,7 @@ class ComposerStaticInitFiles 'OCA\\Files\\Command\\Object\\Delete' => __DIR__ . '/..' . '/../lib/Command/Object/Delete.php', 'OCA\\Files\\Command\\Object\\Get' => __DIR__ . '/..' . '/../lib/Command/Object/Get.php', 'OCA\\Files\\Command\\Object\\Info' => __DIR__ . '/..' . '/../lib/Command/Object/Info.php', + 'OCA\\Files\\Command\\Object\\ListObject' => __DIR__ . '/..' . '/../lib/Command/Object/ListObject.php', 'OCA\\Files\\Command\\Object\\ObjectUtil' => __DIR__ . '/..' . '/../lib/Command/Object/ObjectUtil.php', 'OCA\\Files\\Command\\Object\\Put' => __DIR__ . '/..' . '/../lib/Command/Object/Put.php', 'OCA\\Files\\Command\\Put' => __DIR__ . '/..' . '/../lib/Command/Put.php', diff --git a/apps/files/lib/Command/Object/ListObject.php b/apps/files/lib/Command/Object/ListObject.php new file mode 100644 index 0000000000000..d3b8feae7c79e --- /dev/null +++ b/apps/files/lib/Command/Object/ListObject.php @@ -0,0 +1,119 @@ +setName('files:object:list') + ->setDescription('List all objects in the object store') + ->addOption('bucket', 'b', InputOption::VALUE_REQUIRED, "Bucket to list the objects from, only required in cases where it can't be determined from the config"); + } + + public function execute(InputInterface $input, OutputInterface $output): int { + $objectStore = $this->objectUtils->getObjectStore($input->getOption('bucket'), $output); + if (!$objectStore) { + return self::FAILURE; + } + + if (!$objectStore instanceof IObjectStoreMetaData) { + $output->writeln('Configured object store does currently not support listing objects'); + return self::FAILURE; + } + $outputType = $input->getOption('output'); + $humanOutput = $outputType === self::OUTPUT_FORMAT_PLAIN; + + if (!$humanOutput) { + $output->writeln('['); + } + $objects = $objectStore->listObjects(); + $first = true; + + foreach ($this->chunkIterator($objects, self::CHUNK_SIZE) as $chunk) { + if ($outputType === self::OUTPUT_FORMAT_PLAIN) { + $this->outputChunk($input, $output, $chunk); + } else { + foreach ($chunk as $object) { + if (!$first) { + $output->writeln(','); + } + $row = $this->formatObject($object, $humanOutput); + if ($outputType === self::OUTPUT_FORMAT_JSON_PRETTY) { + $output->write(json_encode($row, JSON_PRETTY_PRINT)); + } else { + $output->write(json_encode($row)); + } + $first = false; + } + } + } + + if (!$humanOutput) { + $output->writeln("\n]"); + } + + return self::SUCCESS; + } + + private function formatObject(array $object, bool $humanOutput): array { + $row = array_merge([ + 'urn' => $object['urn'], + ], ($object['metadata'] ?? [])); + + if ($humanOutput && isset($row['size'])) { + $row['size'] = \OC_Helper::humanFileSize($row['size']); + } + if (isset($row['mtime'])) { + $row['mtime'] = $row['mtime']->format(\DateTimeImmutable::ATOM); + } + return $row; + } + + private function outputChunk(InputInterface $input, OutputInterface $output, iterable $chunk): void { + $result = []; + $humanOutput = $input->getOption('output') === "plain"; + + foreach ($chunk as $object) { + $result[] = $this->formatObject($object, $humanOutput); + } + $this->writeTableInOutputFormat($input, $output, $result); + } + + function chunkIterator(\Iterator $iterator, int $count): \Iterator { + $chunk = []; + + for($i = 0; $iterator->valid(); $i++){ + $chunk[] = $iterator->current(); + $iterator->next(); + if(count($chunk) == $count){ + yield $chunk; + $chunk = []; + } + } + + if(count($chunk)){ + yield $chunk; + } + } +} diff --git a/lib/private/Files/ObjectStore/S3.php b/lib/private/Files/ObjectStore/S3.php index 63c93ca8f3e8f..e970fb6ac145a 100644 --- a/lib/private/Files/ObjectStore/S3.php +++ b/lib/private/Files/ObjectStore/S3.php @@ -118,8 +118,8 @@ public function listObjects(string $prefix = ''): \Iterator { foreach ($result['Contents'] as $object) { yield [ 'urn' => basename($object['Key']), - 'meta' => [ - 'mtime' => strtotime($object['LastModified']), + 'metadata' => [ + 'mtime' => $object['LastModified'], 'etag' => trim($object['ETag'], '"'), 'size' => (int)($object['Size'] ?? $object['ContentLength']), ], diff --git a/lib/public/Files/ObjectStore/IObjectStoreMetaData.php b/lib/public/Files/ObjectStore/IObjectStoreMetaData.php index 48e1290850ca7..8359e83f573ea 100644 --- a/lib/public/Files/ObjectStore/IObjectStoreMetaData.php +++ b/lib/public/Files/ObjectStore/IObjectStoreMetaData.php @@ -30,7 +30,9 @@ public function getObjectMetaData(string $urn): array; * If the object store implementation can do it efficiently, the metadata for each object is also included. * * @param string $prefix - * @return \Iterator + * @return \Iterator + * + * @since 32.0.0 */ public function listObjects(string $prefix = ''): \Iterator; } From f17cf83e16310dab229bd1112729b51f1f7bdc5b Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Thu, 20 Mar 2025 16:33:32 +0100 Subject: [PATCH 3/5] feat: add command to list orphan objects Signed-off-by: Robin Appelman --- apps/files/appinfo/info.xml | 1 + .../composer/composer/autoload_classmap.php | 1 + .../composer/composer/autoload_static.php | 1 + apps/files/lib/Command/Object/ListObject.php | 72 +---------------- apps/files/lib/Command/Object/ObjectUtil.php | 78 ++++++++++++++++++- apps/files/lib/Command/Object/Orphans.php | 73 +++++++++++++++++ 6 files changed, 154 insertions(+), 72 deletions(-) create mode 100644 apps/files/lib/Command/Object/Orphans.php diff --git a/apps/files/appinfo/info.xml b/apps/files/appinfo/info.xml index 5f6f7e52c306e..95f6153e1d710 100644 --- a/apps/files/appinfo/info.xml +++ b/apps/files/appinfo/info.xml @@ -51,6 +51,7 @@ OCA\Files\Command\Object\Put OCA\Files\Command\Object\Info OCA\Files\Command\Object\ListObject + OCA\Files\Command\Object\Orphans diff --git a/apps/files/composer/composer/autoload_classmap.php b/apps/files/composer/composer/autoload_classmap.php index 6489b69179839..a74df7ba3d24f 100644 --- a/apps/files/composer/composer/autoload_classmap.php +++ b/apps/files/composer/composer/autoload_classmap.php @@ -38,6 +38,7 @@ 'OCA\\Files\\Command\\Object\\Info' => $baseDir . '/../lib/Command/Object/Info.php', 'OCA\\Files\\Command\\Object\\ListObject' => $baseDir . '/../lib/Command/Object/ListObject.php', 'OCA\\Files\\Command\\Object\\ObjectUtil' => $baseDir . '/../lib/Command/Object/ObjectUtil.php', + 'OCA\\Files\\Command\\Object\\Orphans' => $baseDir . '/../lib/Command/Object/Orphans.php', 'OCA\\Files\\Command\\Object\\Put' => $baseDir . '/../lib/Command/Object/Put.php', 'OCA\\Files\\Command\\Put' => $baseDir . '/../lib/Command/Put.php', 'OCA\\Files\\Command\\RepairTree' => $baseDir . '/../lib/Command/RepairTree.php', diff --git a/apps/files/composer/composer/autoload_static.php b/apps/files/composer/composer/autoload_static.php index dadac010aea63..1d79f38e35a9a 100644 --- a/apps/files/composer/composer/autoload_static.php +++ b/apps/files/composer/composer/autoload_static.php @@ -53,6 +53,7 @@ class ComposerStaticInitFiles 'OCA\\Files\\Command\\Object\\Info' => __DIR__ . '/..' . '/../lib/Command/Object/Info.php', 'OCA\\Files\\Command\\Object\\ListObject' => __DIR__ . '/..' . '/../lib/Command/Object/ListObject.php', 'OCA\\Files\\Command\\Object\\ObjectUtil' => __DIR__ . '/..' . '/../lib/Command/Object/ObjectUtil.php', + 'OCA\\Files\\Command\\Object\\Orphans' => __DIR__ . '/..' . '/../lib/Command/Object/Orphans.php', 'OCA\\Files\\Command\\Object\\Put' => __DIR__ . '/..' . '/../lib/Command/Object/Put.php', 'OCA\\Files\\Command\\Put' => __DIR__ . '/..' . '/../lib/Command/Put.php', 'OCA\\Files\\Command\\RepairTree' => __DIR__ . '/..' . '/../lib/Command/RepairTree.php', diff --git a/apps/files/lib/Command/Object/ListObject.php b/apps/files/lib/Command/Object/ListObject.php index d3b8feae7c79e..f63eb9c2260fe 100644 --- a/apps/files/lib/Command/Object/ListObject.php +++ b/apps/files/lib/Command/Object/ListObject.php @@ -41,79 +41,9 @@ public function execute(InputInterface $input, OutputInterface $output): int { $output->writeln('Configured object store does currently not support listing objects'); return self::FAILURE; } - $outputType = $input->getOption('output'); - $humanOutput = $outputType === self::OUTPUT_FORMAT_PLAIN; - - if (!$humanOutput) { - $output->writeln('['); - } $objects = $objectStore->listObjects(); - $first = true; - - foreach ($this->chunkIterator($objects, self::CHUNK_SIZE) as $chunk) { - if ($outputType === self::OUTPUT_FORMAT_PLAIN) { - $this->outputChunk($input, $output, $chunk); - } else { - foreach ($chunk as $object) { - if (!$first) { - $output->writeln(','); - } - $row = $this->formatObject($object, $humanOutput); - if ($outputType === self::OUTPUT_FORMAT_JSON_PRETTY) { - $output->write(json_encode($row, JSON_PRETTY_PRINT)); - } else { - $output->write(json_encode($row)); - } - $first = false; - } - } - } - - if (!$humanOutput) { - $output->writeln("\n]"); - } + $this->objectUtils->writeIteratorToOutput($input, $output, $objects, self::CHUNK_SIZE); return self::SUCCESS; } - - private function formatObject(array $object, bool $humanOutput): array { - $row = array_merge([ - 'urn' => $object['urn'], - ], ($object['metadata'] ?? [])); - - if ($humanOutput && isset($row['size'])) { - $row['size'] = \OC_Helper::humanFileSize($row['size']); - } - if (isset($row['mtime'])) { - $row['mtime'] = $row['mtime']->format(\DateTimeImmutable::ATOM); - } - return $row; - } - - private function outputChunk(InputInterface $input, OutputInterface $output, iterable $chunk): void { - $result = []; - $humanOutput = $input->getOption('output') === "plain"; - - foreach ($chunk as $object) { - $result[] = $this->formatObject($object, $humanOutput); - } - $this->writeTableInOutputFormat($input, $output, $result); - } - - function chunkIterator(\Iterator $iterator, int $count): \Iterator { - $chunk = []; - - for($i = 0; $iterator->valid(); $i++){ - $chunk[] = $iterator->current(); - $iterator->next(); - if(count($chunk) == $count){ - yield $chunk; - $chunk = []; - } - } - - if(count($chunk)){ - yield $chunk; - } - } } diff --git a/apps/files/lib/Command/Object/ObjectUtil.php b/apps/files/lib/Command/Object/ObjectUtil.php index c4ab59608fbac..8460e225b61d1 100644 --- a/apps/files/lib/Command/Object/ObjectUtil.php +++ b/apps/files/lib/Command/Object/ObjectUtil.php @@ -8,13 +8,15 @@ namespace OCA\Files\Command\Object; +use OC\Core\Command\Base; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\Files\ObjectStore\IObjectStore; use OCP\IConfig; use OCP\IDBConnection; +use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; -class ObjectUtil { +class ObjectUtil extends Base { public function __construct( private IConfig $config, private IDBConnection $connection, @@ -91,4 +93,78 @@ public function objectExistsInDb(string $object): int|false { return $fileId; } + + public function writeIteratorToOutput(InputInterface $input, OutputInterface $output, \Iterator $objects, int $chunkSize): void { + $outputType = $input->getOption('output'); + $humanOutput = $outputType === Base::OUTPUT_FORMAT_PLAIN; + $first = true; + + if (!$humanOutput) { + $output->writeln('['); + } + + foreach ($this->chunkIterator($objects, $chunkSize) as $chunk) { + if ($outputType === Base::OUTPUT_FORMAT_PLAIN) { + $this->outputChunk($input, $output, $chunk); + } else { + foreach ($chunk as $object) { + if (!$first) { + $output->writeln(','); + } + $row = $this->formatObject($object, $humanOutput); + if ($outputType === Base::OUTPUT_FORMAT_JSON_PRETTY) { + $output->write(json_encode($row, JSON_PRETTY_PRINT)); + } else { + $output->write(json_encode($row)); + } + $first = false; + } + } + } + + if (!$humanOutput) { + $output->writeln("\n]"); + } + } + + private function formatObject(array $object, bool $humanOutput): array { + $row = array_merge([ + 'urn' => $object['urn'], + ], ($object['metadata'] ?? [])); + + if ($humanOutput && isset($row['size'])) { + $row['size'] = \OC_Helper::humanFileSize($row['size']); + } + if (isset($row['mtime'])) { + $row['mtime'] = $row['mtime']->format(\DateTimeImmutable::ATOM); + } + return $row; + } + + private function outputChunk(InputInterface $input, OutputInterface $output, iterable $chunk): void { + $result = []; + $humanOutput = $input->getOption('output') === 'plain'; + + foreach ($chunk as $object) { + $result[] = $this->formatObject($object, $humanOutput); + } + $this->writeTableInOutputFormat($input, $output, $result); + } + + public function chunkIterator(\Iterator $iterator, int $count): \Iterator { + $chunk = []; + + for ($i = 0; $iterator->valid(); $i++) { + $chunk[] = $iterator->current(); + $iterator->next(); + if (count($chunk) == $count) { + yield $chunk; + $chunk = []; + } + } + + if (count($chunk)) { + yield $chunk; + } + } } diff --git a/apps/files/lib/Command/Object/Orphans.php b/apps/files/lib/Command/Object/Orphans.php new file mode 100644 index 0000000000000..22538cf7b911b --- /dev/null +++ b/apps/files/lib/Command/Object/Orphans.php @@ -0,0 +1,73 @@ +query = $connection->getQueryBuilder(); + $this->query->select('fileid') + ->from('filecache') + ->where($this->query->expr()->eq('fileid', $this->query->createParameter('file_id'))); + } + + protected function configure(): void { + parent::configure(); + $this + ->setName('files:object:orphans') + ->setDescription('List all objects in the object store that don\'t have a matching entry in the database') + ->addOption('bucket', 'b', InputOption::VALUE_REQUIRED, "Bucket to list the objects from, only required in cases where it can't be determined from the config"); + } + + public function execute(InputInterface $input, OutputInterface $output): int { + $objectStore = $this->objectUtils->getObjectStore($input->getOption('bucket'), $output); + if (!$objectStore) { + return self::FAILURE; + } + + if (!$objectStore instanceof IObjectStoreMetaData) { + $output->writeln('Configured object store does currently not support listing objects'); + return self::FAILURE; + } + $prefixLength = strlen('urn:oid:'); + + $objects = $objectStore->listObjects('urn:oid:'); + $objects->rewind(); + $orphans = new \CallbackFilterIterator($objects, function (array $object) use ($prefixLength) { + $fileId = (int)substr($object['urn'], $prefixLength); + return !$this->fileIdInDb($fileId); + }); + $orphans = new \ArrayIterator(iterator_to_array($orphans)); + $this->objectUtils->writeIteratorToOutput($input, $output, $orphans, self::CHUNK_SIZE); + + return self::SUCCESS; + } + + private function fileIdInDb(int $fileId): bool { + $this->query->setParameter('file_id', $fileId, IQueryBuilder::PARAM_INT); + $result = $this->query->executeQuery(); + return $result->fetchOne() !== false; + } +} From 7d9655d889b5bacb62b1fc72d7a3d58259f45393 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Tue, 25 Mar 2025 15:17:32 +0100 Subject: [PATCH 4/5] feat: rework object listing Signed-off-by: Robin Appelman --- apps/files/lib/Command/Object/Info.php | 3 +- apps/files/lib/Command/Object/ObjectUtil.php | 51 +++++++++----------- apps/files/lib/Command/Object/Orphans.php | 26 ++++++---- 3 files changed, 42 insertions(+), 38 deletions(-) diff --git a/apps/files/lib/Command/Object/Info.php b/apps/files/lib/Command/Object/Info.php index 5dcc552ea34d0..6748de37cfe2b 100644 --- a/apps/files/lib/Command/Object/Info.php +++ b/apps/files/lib/Command/Object/Info.php @@ -11,6 +11,7 @@ use OC\Core\Command\Base; use OCP\Files\IMimeTypeDetector; use OCP\Files\ObjectStore\IObjectStoreMetaData; +use OCP\Util; use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Input\InputOption; @@ -59,7 +60,7 @@ public function execute(InputInterface $input, OutputInterface $output): int { } if ($input->getOption('output') === 'plain' && isset($meta['size'])) { - $meta['size'] = \OC_Helper::humanFileSize($meta['size']); + $meta['size'] = Util::humanFileSize($meta['size']); } if (isset($meta['mtime'])) { $meta['mtime'] = $meta['mtime']->format(\DateTimeImmutable::ATOM); diff --git a/apps/files/lib/Command/Object/ObjectUtil.php b/apps/files/lib/Command/Object/ObjectUtil.php index 8460e225b61d1..273ad9d6ec93b 100644 --- a/apps/files/lib/Command/Object/ObjectUtil.php +++ b/apps/files/lib/Command/Object/ObjectUtil.php @@ -13,6 +13,7 @@ use OCP\Files\ObjectStore\IObjectStore; use OCP\IConfig; use OCP\IDBConnection; +use OCP\Util; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; @@ -97,32 +98,28 @@ public function objectExistsInDb(string $object): int|false { public function writeIteratorToOutput(InputInterface $input, OutputInterface $output, \Iterator $objects, int $chunkSize): void { $outputType = $input->getOption('output'); $humanOutput = $outputType === Base::OUTPUT_FORMAT_PLAIN; - $first = true; - if (!$humanOutput) { - $output->writeln('['); - } + if ($humanOutput) { + // we can't write tables in a streaming way, so we print them in chunks instead + foreach ($this->chunkIterator($objects, $chunkSize) as $chunk) { + $this->outputChunkHuman($input, $output, $chunk); + } + } else { + $first = true; - foreach ($this->chunkIterator($objects, $chunkSize) as $chunk) { - if ($outputType === Base::OUTPUT_FORMAT_PLAIN) { - $this->outputChunk($input, $output, $chunk); - } else { - foreach ($chunk as $object) { - if (!$first) { - $output->writeln(','); - } - $row = $this->formatObject($object, $humanOutput); - if ($outputType === Base::OUTPUT_FORMAT_JSON_PRETTY) { - $output->write(json_encode($row, JSON_PRETTY_PRINT)); - } else { - $output->write(json_encode($row)); - } - $first = false; + $output->writeln('['); + foreach ($objects as $object) { + if (!$first) { + $output->writeln(','); } + $row = $this->formatObject($object, false); + if ($outputType === self::OUTPUT_FORMAT_JSON_PRETTY) { + $output->write(json_encode($row, JSON_PRETTY_PRINT)); + } else { + $output->write(json_encode($row)); + } + $first = false; } - } - - if (!$humanOutput) { $output->writeln("\n]"); } } @@ -133,7 +130,7 @@ private function formatObject(array $object, bool $humanOutput): array { ], ($object['metadata'] ?? [])); if ($humanOutput && isset($row['size'])) { - $row['size'] = \OC_Helper::humanFileSize($row['size']); + $row['size'] = Util::humanFileSize($row['size']); } if (isset($row['mtime'])) { $row['mtime'] = $row['mtime']->format(\DateTimeImmutable::ATOM); @@ -141,12 +138,10 @@ private function formatObject(array $object, bool $humanOutput): array { return $row; } - private function outputChunk(InputInterface $input, OutputInterface $output, iterable $chunk): void { + private function outputChunkHuman(InputInterface $input, OutputInterface $output, iterable $chunk): void { $result = []; - $humanOutput = $input->getOption('output') === 'plain'; - foreach ($chunk as $object) { - $result[] = $this->formatObject($object, $humanOutput); + $result[] = $this->formatObject($object, true); } $this->writeTableInOutputFormat($input, $output, $result); } @@ -158,12 +153,14 @@ public function chunkIterator(\Iterator $iterator, int $count): \Iterator { $chunk[] = $iterator->current(); $iterator->next(); if (count($chunk) == $count) { + // Got a full chunk, yield and start a new one yield $chunk; $chunk = []; } } if (count($chunk)) { + // Yield the last chunk even if incomplete yield $chunk; } } diff --git a/apps/files/lib/Command/Object/Orphans.php b/apps/files/lib/Command/Object/Orphans.php index 22538cf7b911b..aac05f2d4c051 100644 --- a/apps/files/lib/Command/Object/Orphans.php +++ b/apps/files/lib/Command/Object/Orphans.php @@ -19,18 +19,23 @@ class Orphans extends Base { private const CHUNK_SIZE = 100; - private IQueryBuilder $query; + private ?IQueryBuilder $query = null; public function __construct( private readonly ObjectUtil $objectUtils, - IDBConnection $connection, + private readonly IDBConnection $connection, ) { parent::__construct(); + } - $this->query = $connection->getQueryBuilder(); - $this->query->select('fileid') - ->from('filecache') - ->where($this->query->expr()->eq('fileid', $this->query->createParameter('file_id'))); + private function getQuery(): IQueryBuilder { + if (!$this->query) { + $this->query = $this->connection->getQueryBuilder(); + $this->query->select('fileid') + ->from('filecache') + ->where($this->query->expr()->eq('fileid', $this->query->createParameter('file_id'))); + } + return $this->query; } protected function configure(): void { @@ -54,20 +59,21 @@ public function execute(InputInterface $input, OutputInterface $output): int { $prefixLength = strlen('urn:oid:'); $objects = $objectStore->listObjects('urn:oid:'); - $objects->rewind(); $orphans = new \CallbackFilterIterator($objects, function (array $object) use ($prefixLength) { $fileId = (int)substr($object['urn'], $prefixLength); return !$this->fileIdInDb($fileId); }); - $orphans = new \ArrayIterator(iterator_to_array($orphans)); + $orphans->rewind(); + $this->objectUtils->writeIteratorToOutput($input, $output, $orphans, self::CHUNK_SIZE); return self::SUCCESS; } private function fileIdInDb(int $fileId): bool { - $this->query->setParameter('file_id', $fileId, IQueryBuilder::PARAM_INT); - $result = $this->query->executeQuery(); + $query = $this->getQuery(); + $query->setParameter('file_id', $fileId, IQueryBuilder::PARAM_INT); + $result = $query->executeQuery(); return $result->fetchOne() !== false; } } From 7ce06f47c8724b4b17b957501ab70a92b1c4f2e3 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Fri, 28 Mar 2025 15:23:13 +0100 Subject: [PATCH 5/5] feat: move streaming output helps to command base class Signed-off-by: Robin Appelman --- apps/files/lib/Command/Object/ListObject.php | 3 +- apps/files/lib/Command/Object/ObjectUtil.php | 62 ++------------------ apps/files/lib/Command/Object/Orphans.php | 4 +- core/Command/Base.php | 52 ++++++++++++++++ 4 files changed, 61 insertions(+), 60 deletions(-) diff --git a/apps/files/lib/Command/Object/ListObject.php b/apps/files/lib/Command/Object/ListObject.php index f63eb9c2260fe..5d30232e09f8f 100644 --- a/apps/files/lib/Command/Object/ListObject.php +++ b/apps/files/lib/Command/Object/ListObject.php @@ -42,7 +42,8 @@ public function execute(InputInterface $input, OutputInterface $output): int { return self::FAILURE; } $objects = $objectStore->listObjects(); - $this->objectUtils->writeIteratorToOutput($input, $output, $objects, self::CHUNK_SIZE); + $objects = $this->objectUtils->formatObjects($objects, $input->getOption('output') === self::OUTPUT_FORMAT_PLAIN); + $this->writeStreamingTableInOutputFormat($input, $output, $objects, self::CHUNK_SIZE); return self::SUCCESS; } diff --git a/apps/files/lib/Command/Object/ObjectUtil.php b/apps/files/lib/Command/Object/ObjectUtil.php index 273ad9d6ec93b..5f053c2c42fff 100644 --- a/apps/files/lib/Command/Object/ObjectUtil.php +++ b/apps/files/lib/Command/Object/ObjectUtil.php @@ -8,16 +8,14 @@ namespace OCA\Files\Command\Object; -use OC\Core\Command\Base; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\Files\ObjectStore\IObjectStore; use OCP\IConfig; use OCP\IDBConnection; use OCP\Util; -use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; -class ObjectUtil extends Base { +class ObjectUtil { public function __construct( private IConfig $config, private IDBConnection $connection, @@ -95,36 +93,13 @@ public function objectExistsInDb(string $object): int|false { return $fileId; } - public function writeIteratorToOutput(InputInterface $input, OutputInterface $output, \Iterator $objects, int $chunkSize): void { - $outputType = $input->getOption('output'); - $humanOutput = $outputType === Base::OUTPUT_FORMAT_PLAIN; - - if ($humanOutput) { - // we can't write tables in a streaming way, so we print them in chunks instead - foreach ($this->chunkIterator($objects, $chunkSize) as $chunk) { - $this->outputChunkHuman($input, $output, $chunk); - } - } else { - $first = true; - - $output->writeln('['); - foreach ($objects as $object) { - if (!$first) { - $output->writeln(','); - } - $row = $this->formatObject($object, false); - if ($outputType === self::OUTPUT_FORMAT_JSON_PRETTY) { - $output->write(json_encode($row, JSON_PRETTY_PRINT)); - } else { - $output->write(json_encode($row)); - } - $first = false; - } - $output->writeln("\n]"); + public function formatObjects(\Iterator $objects, bool $humanOutput): \Iterator { + foreach ($objects as $object) { + yield $this->formatObject($object, $humanOutput); } } - private function formatObject(array $object, bool $humanOutput): array { + public function formatObject(array $object, bool $humanOutput): array { $row = array_merge([ 'urn' => $object['urn'], ], ($object['metadata'] ?? [])); @@ -137,31 +112,4 @@ private function formatObject(array $object, bool $humanOutput): array { } return $row; } - - private function outputChunkHuman(InputInterface $input, OutputInterface $output, iterable $chunk): void { - $result = []; - foreach ($chunk as $object) { - $result[] = $this->formatObject($object, true); - } - $this->writeTableInOutputFormat($input, $output, $result); - } - - public function chunkIterator(\Iterator $iterator, int $count): \Iterator { - $chunk = []; - - for ($i = 0; $iterator->valid(); $i++) { - $chunk[] = $iterator->current(); - $iterator->next(); - if (count($chunk) == $count) { - // Got a full chunk, yield and start a new one - yield $chunk; - $chunk = []; - } - } - - if (count($chunk)) { - // Yield the last chunk even if incomplete - yield $chunk; - } - } } diff --git a/apps/files/lib/Command/Object/Orphans.php b/apps/files/lib/Command/Object/Orphans.php index aac05f2d4c051..f7132540fc82f 100644 --- a/apps/files/lib/Command/Object/Orphans.php +++ b/apps/files/lib/Command/Object/Orphans.php @@ -63,9 +63,9 @@ public function execute(InputInterface $input, OutputInterface $output): int { $fileId = (int)substr($object['urn'], $prefixLength); return !$this->fileIdInDb($fileId); }); - $orphans->rewind(); - $this->objectUtils->writeIteratorToOutput($input, $output, $orphans, self::CHUNK_SIZE); + $orphans = $this->objectUtils->formatObjects($orphans, $input->getOption('output') === self::OUTPUT_FORMAT_PLAIN); + $this->writeStreamingTableInOutputFormat($input, $output, $orphans, self::CHUNK_SIZE); return self::SUCCESS; } diff --git a/core/Command/Base.php b/core/Command/Base.php index b915ae2ae4a3a..c9b6337b64a9a 100644 --- a/core/Command/Base.php +++ b/core/Command/Base.php @@ -88,6 +88,58 @@ protected function writeTableInOutputFormat(InputInterface $input, OutputInterfa } } + protected function writeStreamingTableInOutputFormat(InputInterface $input, OutputInterface $output, \Iterator $items, int $tableGroupSize): void { + switch ($input->getOption('output')) { + case self::OUTPUT_FORMAT_JSON: + case self::OUTPUT_FORMAT_JSON_PRETTY: + $this->writeStreamingJsonArray($input, $output, $items); + break; + default: + foreach ($this->chunkIterator($items, $tableGroupSize) as $chunk) { + $this->writeTableInOutputFormat($input, $output, $chunk); + } + break; + } + } + + protected function writeStreamingJsonArray(InputInterface $input, OutputInterface $output, \Iterator $items): void { + $first = true; + $outputType = $input->getOption('output'); + + $output->writeln('['); + foreach ($items as $item) { + if (!$first) { + $output->writeln(','); + } + if ($outputType === self::OUTPUT_FORMAT_JSON_PRETTY) { + $output->write(json_encode($item, JSON_PRETTY_PRINT)); + } else { + $output->write(json_encode($item)); + } + $first = false; + } + $output->writeln("\n]"); + } + + public function chunkIterator(\Iterator $iterator, int $count): \Iterator { + $chunk = []; + + for ($i = 0; $iterator->valid(); $i++) { + $chunk[] = $iterator->current(); + $iterator->next(); + if (count($chunk) == $count) { + // Got a full chunk, yield and start a new one + yield $chunk; + $chunk = []; + } + } + + if (count($chunk)) { + // Yield the last chunk even if incomplete + yield $chunk; + } + } + /** * @param mixed $item