Merge pull request #32708 from nextcloud/unencrypted-size-24

[24] store unencrypted size in the unencrypted_size column

Author: blizzz

lib/private/Files/Cache/Cache.php

@@ -37,6 +37,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>
  *
  */
+
 namespace OC\Files\Cache;
 
 use Doctrine\DBAL\Exception\UniqueConstraintViolationException;
@@ -188,6 +189,7 @@ public static function cacheEntryFromData($data, IMimeTypeLoader $mimetypeLoader
 		$data['fileid'] = (int)$data['fileid'];
 		$data['parent'] = (int)$data['parent'];
 		$data['size'] = 0 + $data['size'];
+		$data['unencrypted_size'] = 0 + ($data['unencrypted_size'] ?? 0);
 		$data['mtime'] = (int)$data['mtime'];
 		$data['storage_mtime'] = (int)$data['storage_mtime'];
 		$data['encryptedVersion'] = (int)$data['encrypted'];
@@ -428,7 +430,7 @@ public function update($id, array $data) {
 	protected function normalizeData(array $data): array {
 		$fields = [
 			'path', 'parent', 'name', 'mimetype', 'size', 'mtime', 'storage_mtime', 'encrypted',
-			'etag', 'permissions', 'checksum', 'storage'];
+			'etag', 'permissions', 'checksum', 'storage', 'unencrypted_size'];
 		$extensionFields = ['metadata_etag', 'creation_time', 'upload_time'];
 
 		$doNotCopyStorageMTime = false;
@@ -873,27 +875,56 @@ public function calculateFolderSize($path, $entry = null) {
 			$id = $entry['fileid'];
 
 			$query = $this->getQueryBuilder();
-			$query->selectAlias($query->func()->sum('size'), 'f1')
-				->selectAlias($query->func()->min('size'), 'f2')
+			$query->select('size', 'unencrypted_size')
 				->from('filecache')
-				->whereStorageId($this->getNumericStorageId())
 				->whereParent($id);
 
 			$result = $query->execute();
-			$row = $result->fetch();
+			$rows = $result->fetchAll();
 			$result->closeCursor();
 
-			if ($row) {
-				[$sum, $min] = array_values($row);
+			if ($rows) {
+				$sizes = array_map(function (array $row) {
+					return (int)$row['size'];
+				}, $rows);
+				$unencryptedOnlySizes = array_map(function (array $row) {
+					return (int)$row['unencrypted_size'];
+				}, $rows);
+				$unencryptedSizes = array_map(function (array $row) {
+					return (int)(($row['unencrypted_size'] > 0) ? $row['unencrypted_size']: $row['size']);
+				}, $rows);
+
+				$sum = array_sum($sizes);
+				$min = min($sizes);
+
+				$unencryptedSum = array_sum($unencryptedSizes);
+				$unencryptedMin = min($unencryptedSizes);
+				$unencryptedMax = max($unencryptedOnlySizes);
+
 				$sum = 0 + $sum;
 				$min = 0 + $min;
 				if ($min === -1) {
 					$totalSize = $min;
 				} else {
 					$totalSize = $sum;
 				}
+				if ($unencryptedMin === -1 || $min === -1) {
+					$unencryptedTotal = $unencryptedMin;
+				} else {
+					$unencryptedTotal = $unencryptedSum;
+				}
 				if ($entry['size'] !== $totalSize) {
-					$this->update($id, ['size' => $totalSize]);
+					// only set unencrypted size for a folder if any child entries have it set
+					if ($unencryptedMax > 0) {
+						$this->update($id, [
+							'size' => $totalSize,
+							'unencrypted_size' => $unencryptedTotal,
+						]);
+					} else {
+						$this->update($id, [
+							'size' => $totalSize,
+						]);
+					}
 				}
 			}
 		}

lib/private/Files/Cache/CacheEntry.php

@@ -132,4 +132,12 @@ public function getData() {
 	public function __clone() {
 		$this->data = array_merge([], $this->data);
 	}
+
+	public function getUnencryptedSize(): int {
+		if (isset($this->data['unencrypted_size']) && $this->data['unencrypted_size'] > 0) {
+			return $this->data['unencrypted_size'];
+		} else {
+			return $this->data['size'];
+		}
+	}
 }

lib/private/Files/Cache/CacheQueryBuilder.php

@@ -44,7 +44,7 @@ public function __construct(IDBConnection $connection, SystemConfig $systemConfi
 	public function selectFileCache(string $alias = null) {
 		$name = $alias ? $alias : 'filecache';
 		$this->select("$name.fileid", 'storage', 'path', 'path_hash', "$name.parent", 'name', 'mimetype', 'mimepart', 'size', 'mtime',
-			'storage_mtime', 'encrypted', 'etag', 'permissions', 'checksum', 'metadata_etag', 'creation_time', 'upload_time')
+			'storage_mtime', 'encrypted', 'etag', 'permissions', 'checksum', 'metadata_etag', 'creation_time', 'upload_time', 'unencrypted_size')
 			->from('filecache', $name)
 			->leftJoin($name, 'filecache_extended', 'fe', $this->expr()->eq("$name.fileid", 'fe.fileid'));
 

lib/private/Files/Cache/Propagator.php

@@ -24,6 +24,8 @@
 
 namespace OC\Files\Cache;
 
+use OC\DB\QueryBuilder\QueryFunction;
+use OC\Files\Storage\Wrapper\Encryption;
 use OCP\DB\QueryBuilder\IQueryBuilder;
 use OCP\Files\Cache\IPropagator;
 use OCP\Files\Storage\IReliableEtagStorage;
@@ -113,6 +115,20 @@ public function propagateChange($internalPath, $time, $sizeDifference = 0) {
 				->andWhere($builder->expr()->in('path_hash', $hashParams))
 				->andWhere($builder->expr()->gt('size', $builder->expr()->literal(-1, IQueryBuilder::PARAM_INT)));
 
+			if ($this->storage->instanceOfStorage(Encryption::class)) {
+				// in case of encryption being enabled after some files are already uploaded, some entries will have an unencrypted_size of 0 and a non-zero size
+				$eq = $builder->expr()->eq('unencrypted_size', $builder->expr()->literal(0, IQueryBuilder::PARAM_INT));
+				$sizeColumn = $builder->getColumnName('size');
+				$unencryptedSizeColumn = $builder->getColumnName('unencrypted_size');
+				$builder->set('unencrypted_size', $builder->func()->greatest(
+					$builder->func()->add(
+						new QueryFunction("CASE WHEN $eq THEN $unencryptedSizeColumn ELSE $sizeColumn END"),
+						$builder->createNamedParameter($sizeDifference)
+					),
+					$builder->createNamedParameter(-1, IQueryBuilder::PARAM_INT)
+				));
+			}
+
 			$builder->execute();
 		}
 	}

lib/private/Files/FileInfo.php

@@ -101,7 +101,11 @@ public function __construct($path, $storage, $internalPath, $data, $mount, $owne
 		$this->data = $data;
 		$this->mount = $mount;
 		$this->owner = $owner;
-		$this->rawSize = $this->data['size'] ?? 0;
+		if (isset($this->data['unencrypted_size'])) {
+			$this->rawSize = $this->data['unencrypted_size'];
+		} else {
+			$this->rawSize = $this->data['size'] ?? 0;
+		}
 	}
 
 	public function offsetSet($offset, $value): void {
@@ -208,7 +212,12 @@ public function getEtag() {
 	public function getSize($includeMounts = true) {
 		if ($includeMounts) {
 			$this->updateEntryfromSubMounts();
-			return isset($this->data['size']) ? 0 + $this->data['size'] : 0;
+
+			if (isset($this->data['unencrypted_size']) && $this->data['unencrypted_size'] > 0) {
+				return $this->data['unencrypted_size'];
+			} else {
+				return isset($this->data['size']) ? 0 + $this->data['size'] : 0;
+			}
 		} else {
 			return $this->rawSize;
 		}
@@ -386,7 +395,19 @@ private function updateEntryfromSubMounts() {
 	 * @param string $entryPath full path of the child entry
 	 */
 	public function addSubEntry($data, $entryPath) {
-		$this->data['size'] += isset($data['size']) ? $data['size'] : 0;
+		if (!$data) {
+			return;
+		}
+		$hasUnencryptedSize = isset($data['unencrypted_size']) && $data['unencrypted_size'] > 0;
+		if ($hasUnencryptedSize) {
+			$subSize = $data['unencrypted_size'];
+		} else {
+			$subSize = $data['size'] ?: 0;
+		}
+		$this->data['size'] += $subSize;
+		if ($hasUnencryptedSize) {
+			$this->data['unencrypted_size'] += $subSize;
+		}
 		if (isset($data['mtime'])) {
 			$this->data['mtime'] = max($this->data['mtime'], $data['mtime']);
 		}