Skip to content

Commit 6d38880

Browse files
CarlSchwanCarlSchwan
committed
Cache locally the filecache information
This on one side might improve the performance by doing less requests to the database but more importantly this decrease the likeliness of read after write errors in a cluster setup since we don't requests the data from the database that might be outdated. Currently this only cache get/put/insert/update and for copy/move/propagator this clear the cache. Signed-off-by: Carl Schwan <carl@carlschwan.eu>
1 parent cf4c77e commit 6d38880

File tree

3 files changed

+106
-60
lines changed

3 files changed

+106
-60
lines changed

‎lib/private/Files/Cache/Cache.php‎

Lines changed: 103 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@
5959
use OCP\Files\Storage\IStorage;
6060
use OCP\IDBConnection;
6161
use Psr\Log\LoggerInterface;
62+
use OC\Cache\CappedMemoryCache;
6263

6364
/**
6465
* Metadata cache for a storage
@@ -76,41 +77,21 @@ class Cache implements ICache {
7677
}
7778

7879
/**
79-
* @var array partial data for the cache
80+
* Internal cache that allows to store temporarely data without having to
81+
* fetch them from the database
8082
*/
81-
protected $partial = [];
83+
private CappedMemoryCache $internalCache;
84+
85+
/** @var array partial data for the cache */
86+
protected array $partial = [];
87+
protected string $storageId;
88+
private IStorage $storage;
89+
protected Storage $storageCache;
90+
protected IMimeTypeLoader $mimetypeLoader;
91+
protected IDBConnection $connection;
92+
protected IEventDispatcher $eventDispatcher;
93+
protected QuerySearchHelper $querySearchHelper;
8294

83-
/**
84-
* @var string
85-
*/
86-
protected $storageId;
87-
88-
private $storage;
89-
90-
/**
91-
* @var Storage $storageCache
92-
*/
93-
protected $storageCache;
94-
95-
/** @var IMimeTypeLoader */
96-
protected $mimetypeLoader;
97-
98-
/**
99-
* @var IDBConnection
100-
*/
101-
protected $connection;
102-
103-
/**
104-
* @var IEventDispatcher
105-
*/
106-
protected $eventDispatcher;
107-
108-
/** @var QuerySearchHelper */
109-
protected $querySearchHelper;
110-
111-
/**
112-
* @param IStorage $storage
113-
*/
11495
public function __construct(IStorage $storage) {
11596
$this->storageId = $storage->getId();
11697
$this->storage = $storage;
@@ -123,9 +104,10 @@ public function __construct(IStorage $storage) {
123104
$this->connection = \OC::$server->getDatabaseConnection();
124105
$this->eventDispatcher = \OC::$server->get(IEventDispatcher::class);
125106
$this->querySearchHelper = \OC::$server->query(QuerySearchHelper::class);
107+
$this->internalCache = new CappedMemoryCache();
126108
}
127109

128-
protected function getQueryBuilder() {
110+
protected function getQueryBuilder(): CacheQueryBuilder {
129111
return new CacheQueryBuilder(
130112
$this->connection,
131113
\OC::$server->getSystemConfig(),
@@ -143,12 +125,18 @@ public function getNumericStorageId() {
143125
}
144126

145127
/**
146-
* get the stored metadata of a file or folder
128+
* Get the stored metadata of a file or folder
147129
*
148-
* @param string | int $file either the path of a file or folder or the file id for a file or folder
130+
* @param string|int $file either the path of a file or folder or the file id for a file or folder
149131
* @return ICacheEntry|false the cache entry as array of false if the file is not found in the cache
150132
*/
151133
public function get($file) {
134+
// Try fetching from memory cache first
135+
if ($this->internalCache->hasKey($file)) {
136+
return $this->internalCache->get($file);
137+
}
138+
139+
// Fetch cache entry from database
152140
$query = $this->getQueryBuilder();
153141
$query->selectFileCache();
154142

@@ -166,25 +154,22 @@ public function get($file) {
166154
$data = $result->fetch();
167155
$result->closeCursor();
168156

169-
//merge partial data
157+
// If no cache entry is found, return the partially saved in-memory data
170158
if (!$data && is_string($file) && isset($this->partial[$file])) {
171159
return $this->partial[$file];
172-
} elseif (!$data) {
173-
return $data;
174-
} else {
175-
return self::cacheEntryFromData($data, $this->mimetypeLoader);
176160
}
161+
162+
if (!$data) {
163+
return false;
164+
}
165+
return self::cacheEntryFromData($data, $this->mimetypeLoader);
177166
}
178167

179168
/**
180169
* Create a CacheEntry from database row
181-
*
182-
* @param array $data
183-
* @param IMimeTypeLoader $mimetypeLoader
184-
* @return CacheEntry
185170
*/
186-
public static function cacheEntryFromData($data, IMimeTypeLoader $mimetypeLoader) {
187-
//fix types
171+
public static function cacheEntryFromData(array $data, IMimeTypeLoader $mimetypeLoader): CacheEntry {
172+
// fix types
188173
$data['fileid'] = (int)$data['fileid'];
189174
$data['parent'] = (int)$data['parent'];
190175
$data['size'] = 0 + $data['size'];
@@ -221,7 +206,7 @@ public function getFolderContents($folder) {
221206
}
222207

223208
/**
224-
* get the metadata of all files stored in $folder
209+
* Get the metadata of all files stored in $folder
225210
*
226211
* @param int $fileId the file id of the folder
227212
* @return ICacheEntry[]
@@ -237,15 +222,18 @@ public function getFolderContentsById($fileId) {
237222
$files = $result->fetchAll();
238223
$result->closeCursor();
239224

240-
return array_map(function (array $data) {
241-
return self::cacheEntryFromData($data, $this->mimetypeLoader);
225+
return array_map(function (array $data): ICacheEntry {
226+
$cacheEntry = self::cacheEntryFromData($data, $this->mimetypeLoader);
227+
$this->internalCache->set($cacheEntry->getId(), $cacheEntry);
228+
$this->internalCache->set($cacheEntry->getPath(), $cacheEntry);
229+
return $cacheEntry;
242230
}, $files);
243231
}
244232
return [];
245233
}
246234

247235
/**
248-
* insert or update meta data for a file or folder
236+
* Insert or update meta data for a file or folder
249237
*
250238
* @param string $file
251239
* @param array $data
@@ -263,7 +251,7 @@ public function put($file, array $data) {
263251
}
264252

265253
/**
266-
* insert meta data for a new file or folder
254+
* Insert meta data for a new file or folder
267255
*
268256
* @param string $file
269257
* @param array $data
@@ -316,10 +304,34 @@ public function insert($file, array $data) {
316304
$query->setValue('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT));
317305
foreach ($extensionValues as $column => $value) {
318306
$query->setValue($column, $query->createNamedParameter($value));
307+
$values[$column] = $value;
319308
}
320309
$query->execute();
321310
}
322311

312+
// Add possible missing values
313+
$values['fileid'] = $fileId;
314+
$valuesDefault = [
315+
'encrypted' => 0,
316+
'storage_mtime' => 0,
317+
'permissions' => 0,
318+
'etag' => null,
319+
'checksum' => null,
320+
'metadata_etag' => null,
321+
'creation_time' => null,
322+
'upload_time' => null,
323+
];
324+
foreach ($valuesDefault as $key => $default) {
325+
if (!array_key_exists($key, $values)) {
326+
$values[$key] = $default; // not set => set it to 0 as default
327+
}
328+
}
329+
// FIXME maybe we should fix the test instead
330+
// $values['storageid'] = (string)$values['storageid'];
331+
332+
// Save in in-memory cache
333+
$this->internalCache->set($fileId, self::cacheEntryFromData($values, $this->mimetypeLoader));
334+
323335
$event = new CacheEntryInsertedEvent($this->storage, $file, $fileId, $storageId);
324336
$this->eventDispatcher->dispatch(CacheInsertEvent::class, $event);
325337
$this->eventDispatcher->dispatchTyped($event);
@@ -360,6 +372,7 @@ public function update($id, array $data) {
360372
}
361373

362374
[$values, $extensionValues] = $this->normalizeData($data);
375+
$valuesToUpdateInCache = [];
363376

364377
if (count($values)) {
365378
$query = $this->getQueryBuilder();
@@ -375,6 +388,7 @@ public function update($id, array $data) {
375388

376389
foreach ($values as $key => $value) {
377390
$query->set($key, $query->createNamedParameter($value));
391+
$valuesToUpdateInCache[$key] = $value;
378392
}
379393

380394
$query->execute();
@@ -388,6 +402,7 @@ public function update($id, array $data) {
388402
$query->setValue('fileid', $query->createNamedParameter($id, IQueryBuilder::PARAM_INT));
389403
foreach ($extensionValues as $column => $value) {
390404
$query->setValue($column, $query->createNamedParameter($value));
405+
$valuesToUpdateInCache[$column] = $value;
391406
}
392407

393408
$query->execute();
@@ -413,6 +428,16 @@ public function update($id, array $data) {
413428
$path = $this->getPathById($id);
414429
// path can still be null if the file doesn't exist
415430
if ($path !== null) {
431+
if ($this->internalCache->hasKey($id)) {
432+
// Only update when we already have an entry in the cache
433+
// otherwise we don't have all the values
434+
$cacheEntry = $this->internalCache->get($id);
435+
foreach ($valuesToUpdateInCache as $key => $value) {
436+
$cacheEntry[$key] = $value;
437+
}
438+
$this->internalCache->set($id, $cacheEntry);
439+
}
440+
416441
$event = new CacheEntryUpdatedEvent($this->storage, $path, $id, $this->getNumericStorageId());
417442
$this->eventDispatcher->dispatch(CacheUpdateEvent::class, $event);
418443
$this->eventDispatcher->dispatchTyped($event);
@@ -469,7 +494,7 @@ protected function normalizeData(array $data): array {
469494
}
470495

471496
/**
472-
* get the file id for a file
497+
* Get the file id for a file
473498
*
474499
* A file id is a numeric id for a file or folder that's unique within an owncloud instance which stays the same for the lifetime of a file
475500
*
@@ -482,6 +507,10 @@ public function getId($file) {
482507
// normalize file
483508
$file = $this->normalize($file);
484509

510+
if ($this->internalCache->hasKey($file)) {
511+
return $this->internalCache->get($file)->getId();
512+
}
513+
485514
$query = $this->getQueryBuilder();
486515
$query->select('fileid')
487516
->from('filecache')
@@ -505,6 +534,9 @@ public function getParentId($file) {
505534
if ($file === '') {
506535
return -1;
507536
} else {
537+
if ($this->internalCache->hasKey($file)) {
538+
return $this->internalCache->get($file)['parent'];
539+
}
508540
$parent = $this->getParentPath($file);
509541
return (int)$this->getId($parent);
510542
}
@@ -529,9 +561,9 @@ public function inCache($file) {
529561
}
530562

531563
/**
532-
* remove a file or folder from the cache
564+
* Remove a file or folder from the cache
533565
*
534-
* when removing a folder from the cache all files and folders inside the folder will be removed as well
566+
* When removing a folder from the cache all files and folders inside the folder will be removed as well.
535567
*
536568
* @param string $file
537569
*/
@@ -549,14 +581,22 @@ public function remove($file) {
549581
->whereFileId($entry->getId());
550582
$query->execute();
551583

552-
if ($entry->getMimeType() == FileInfo::MIMETYPE_FOLDER) {
584+
if ($entry->getMimeType() === FileInfo::MIMETYPE_FOLDER) {
553585
$this->removeChildren($entry);
586+
$this->internalCache->clear();
587+
} else {
588+
$this->internalCache->remove($entry->getId());
589+
$this->internalCache->remove($entry->getPath());
554590
}
555591

556592
$this->eventDispatcher->dispatchTyped(new CacheEntryRemovedEvent($this->storage, $entry->getPath(), $entry->getId(), $this->getNumericStorageId()));
557593
}
558594
}
559595

596+
public function clearInternalCache(): void {
597+
$this->internalCache->clear();
598+
}
599+
560600
/**
561601
* Get all sub folders of a folder
562602
*
@@ -565,8 +605,8 @@ public function remove($file) {
565605
*/
566606
private function getSubFolders(ICacheEntry $entry) {
567607
$children = $this->getFolderContentsById($entry->getId());
568-
return array_filter($children, function ($child) {
569-
return $child->getMimeType() == FileInfo::MIMETYPE_FOLDER;
608+
return array_filter($children, function ($child):bool {
609+
return $child->getMimeType() === FileInfo::MIMETYPE_FOLDER;
570610
});
571611
}
572612

@@ -576,7 +616,7 @@ private function getSubFolders(ICacheEntry $entry) {
576616
* @param ICacheEntry $entry the cache entry of the folder to remove the children of
577617
* @throws \OC\DatabaseException
578618
*/
579-
private function removeChildren(ICacheEntry $entry) {
619+
private function removeChildren(ICacheEntry $entry): void {
580620
$parentIds = [$entry->getId()];
581621
$queue = [$entry->getId()];
582622

@@ -690,6 +730,7 @@ public function moveFromCache(ICache $sourceCache, $sourcePath, $targetPath) {
690730

691731
try {
692732
$query->execute();
733+
$this->internalCache->clear();
693734
} catch (\OC\DatabaseException $e) {
694735
$this->connection->rollBack();
695736
throw $e;
@@ -705,6 +746,9 @@ public function moveFromCache(ICache $sourceCache, $sourcePath, $targetPath) {
705746
->set('parent', $query->createNamedParameter($newParentId, IQueryBuilder::PARAM_INT))
706747
->whereFileId($sourceId);
707748
$query->execute();
749+
$this->internalCache->clear();
750+
$this->internalCache->remove($sourceId);
751+
$this->internalCache->remove($sourceData->getPath());
708752

709753
$this->connection->commit();
710754

‎lib/private/Files/Cache/Propagator.php‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,7 @@ public function propagateChange($internalPath, $time, $sizeDifference = 0) {
111111

112112
$builder->execute();
113113
}
114+
$this->storage->getCache()->clearInternalCache();
114115
}
115116

116117
protected function getParents($path) {
@@ -195,5 +196,6 @@ public function commitBatch() {
195196
$this->batch = [];
196197

197198
$this->connection->commit();
199+
$this->storage->getCache()->clearInternalCache();
198200
}
199201
}

‎tests/lib/Files/ViewTest.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -217,7 +217,7 @@ public function testCacheAPI() {
217217
$this->assertFalse($cachedData['encrypted']);
218218
$id = $rootView->putFileInfo('/foo.txt', ['encrypted' => true]);
219219
$cachedData = $rootView->getFileInfo('/foo.txt');
220-
$this->assertTrue($cachedData['encrypted']);
220+
$this->assertTrue((bool)$cachedData['encrypted']);
221221
$this->assertEquals($cachedData['fileid'], $id);
222222

223223
$this->assertFalse($rootView->getFileInfo('/non/existing'));

0 commit comments

Comments
 (0)