Skip to content

[wip] Filecache chunking #43576

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
icewind1991 wants to merge 7 commits into from
Closed

[wip] Filecache chunking #43576

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
216a92e
feat: add class for (future) shared access to the filecache
icewind1991 Feb 14, 2024
647e64d
expose escapeLikeParameter trough query builder
icewind1991 Feb 14, 2024
b3b0c14
block access to the filecache tables by default
icewind1991 Feb 14, 2024
1611232
adjust share provider wip
icewind1991 Feb 20, 2024
f481a4a
refactor: don't join on filecache in usermountcache
icewind1991 Mar 7, 2024
e7c8840
transform favorite, tag and systemtag search queries to fileid lists
icewind1991 Mar 1, 2024
4c6aabf
register DI alias for IFileAccess
icewind1991 May 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions lib/composer/composer/autoload_classmap.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1405,6 +1405,7 @@
'OC\\Files\\Cache\\CacheDependencies' => $baseDir . '/lib/private/Files/Cache/CacheDependencies.php',
'OC\\Files\\Cache\\CacheEntry' => $baseDir . '/lib/private/Files/Cache/CacheEntry.php',
'OC\\Files\\Cache\\CacheQueryBuilder' => $baseDir . '/lib/private/Files/Cache/CacheQueryBuilder.php',
'OC\\Files\\Cache\\Database' => $baseDir . '/lib/private/Files/Cache/Database.php',
'OC\\Files\\Cache\\FailedCache' => $baseDir . '/lib/private/Files/Cache/FailedCache.php',
'OC\\Files\\Cache\\FileAccess' => $baseDir . '/lib/private/Files/Cache/FileAccess.php',
'OC\\Files\\Cache\\HomeCache' => $baseDir . '/lib/private/Files/Cache/HomeCache.php',
Expand Down
1 change: 1 addition & 0 deletions lib/composer/composer/autoload_static.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1446,6 +1446,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
'OC\\Files\\Cache\\CacheDependencies' => __DIR__ . '/../../..' . '/lib/private/Files/Cache/CacheDependencies.php',
'OC\\Files\\Cache\\CacheEntry' => __DIR__ . '/../../..' . '/lib/private/Files/Cache/CacheEntry.php',
'OC\\Files\\Cache\\CacheQueryBuilder' => __DIR__ . '/../../..' . '/lib/private/Files/Cache/CacheQueryBuilder.php',
'OC\\Files\\Cache\\Database' => __DIR__ . '/../../..' . '/lib/private/Files/Cache/Database.php',
'OC\\Files\\Cache\\FailedCache' => __DIR__ . '/../../..' . '/lib/private/Files/Cache/FailedCache.php',
'OC\\Files\\Cache\\FileAccess' => __DIR__ . '/../../..' . '/lib/private/Files/Cache/FileAccess.php',
'OC\\Files\\Cache\\HomeCache' => __DIR__ . '/../../..' . '/lib/private/Files/Cache/HomeCache.php',
Expand Down
49 changes: 49 additions & 0 deletions lib/private/DB/QueryBuilder/QueryBuilder.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,19 @@ class QueryBuilder implements IQueryBuilder {
/** @var string */
protected $lastInsertedTable;

/**
* Tables that require special attention and thus can't be queried by default
*
* @var list<string>
*/
protected array $shardedTables = [
'filecache',
'filecache_extended',
'files_metadata'
];

protected bool $sharded = false;

/**
* Initializes a new QueryBuilder.
*
Expand Down Expand Up @@ -662,6 +675,17 @@ public function insert($insert = null) {
return $this;
}

/**
* @param string $table
* @return void
* @throws \Exception
*/
private function checkTableAccess(string $table) {
if (in_array($table, $this->shardedTables) !== $this->sharded) {
throw new \Exception("current query isn't allowed to access the $table table");
}
}

/**
* Creates and adds a query root corresponding to the table identified by the
* given alias, forming a cartesian product with any existing query roots.
Expand All @@ -678,6 +702,7 @@ public function insert($insert = null) {
* @return $this This QueryBuilder instance.
*/
public function from($from, $alias = null) {
$this->checkTableAccess($from);

Check failure

Code scanning / Psalm

ImplicitToStringCast

Argument 1 of OC\DB\QueryBuilder\QueryBuilder::checkTableAccess expects string, but OCP\DB\QueryBuilder\IQueryFunction|string provided with a __toString method
$this->queryBuilder->from(
$this->getTableName($from),
$this->quoteAlias($alias)
Expand All @@ -704,6 +729,8 @@ public function from($from, $alias = null) {
* @return $this This QueryBuilder instance.
*/
public function join($fromAlias, $join, $alias, $condition = null) {
$this->checkTableAccess($join);

$this->queryBuilder->join(
$this->quoteAlias($fromAlias),
$this->getTableName($join),
Expand Down Expand Up @@ -732,6 +759,8 @@ public function join($fromAlias, $join, $alias, $condition = null) {
* @return $this This QueryBuilder instance.
*/
public function innerJoin($fromAlias, $join, $alias, $condition = null) {
$this->checkTableAccess($join);

$this->queryBuilder->innerJoin(
$this->quoteAlias($fromAlias),
$this->getTableName($join),
Expand Down Expand Up @@ -760,6 +789,8 @@ public function innerJoin($fromAlias, $join, $alias, $condition = null) {
* @return $this This QueryBuilder instance.
*/
public function leftJoin($fromAlias, $join, $alias, $condition = null) {
$this->checkTableAccess($join);

$this->queryBuilder->leftJoin(
$this->quoteAlias($fromAlias),
$this->getTableName($join),
Expand Down Expand Up @@ -788,6 +819,8 @@ public function leftJoin($fromAlias, $join, $alias, $condition = null) {
* @return $this This QueryBuilder instance.
*/
public function rightJoin($fromAlias, $join, $alias, $condition = null) {
$this->checkTableAccess($join);

$this->queryBuilder->rightJoin(
$this->quoteAlias($fromAlias),
$this->getTableName($join),
Expand Down Expand Up @@ -1337,4 +1370,20 @@ public function quoteAlias($alias) {

return $this->helper->quoteColumnName($alias);
}

public function escapeLikeParameter(string $parameter): string {
return $this->connection->escapeLikeParameter($parameter);
}

/**
* Mark the query as accessing the sharded tables
*
* Proper attention needs to be given to ensure that all requirements for accessing the sharded tables
*
* @param bool $sharded
* @return void
*/
public function setSharded(bool $sharded): void {
$this->sharded = $sharded;
}
}
71 changes: 23 additions & 48 deletions lib/private/Files/Cache/Cache.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@
use OCP\Files\Search\ISearchQuery;
use OCP\Files\Storage\IStorage;
use OCP\FilesMetadata\IFilesMetadataManager;
use OCP\IDBConnection;
use OCP\Server;
use OCP\Util;
use Psr\Log\LoggerInterface;

Expand All @@ -88,7 +88,7 @@ class Cache implements ICache {
protected string $storageId;
protected Storage $storageCache;
protected IMimeTypeLoader$mimetypeLoader;
protected IDBConnection $connection;
protected CacheDatabase $cacheDb;
protected SystemConfig $systemConfig;
protected LoggerInterface $logger;
protected QuerySearchHelper $querySearchHelper;
Expand All @@ -106,25 +106,20 @@ public function __construct(
$this->storageId = md5($this->storageId);
}
if (!$dependencies) {
$dependencies = \OC::$server->get(CacheDependencies::class);
$dependencies = Server::get(CacheDependencies::class);
}
$this->storageCache = new Storage($this->storage, true, $dependencies->getConnection());
$this->mimetypeLoader = $dependencies->getMimeTypeLoader();
$this->connection = $dependencies->getConnection();
$this->cacheDb = $dependencies->getCacheDb();
$this->systemConfig = $dependencies->getSystemConfig();
$this->logger = $dependencies->getLogger();
$this->querySearchHelper = $dependencies->getQuerySearchHelper();
$this->eventDispatcher = $dependencies->getEventDispatcher();
$this->metadataManager = $dependencies->getMetadataManager();
}

protected function getQueryBuilder() {
return new CacheQueryBuilder(
$this->connection,
$this->systemConfig,
$this->logger,
$this->metadataManager,
);
public function getQueryBuilder() {
return $this->cacheDb->queryForStorageId($this->getNumericStorageId());
}

public function getStorageCache(): Storage {
Expand All @@ -149,7 +144,7 @@ public function getNumericStorageId() {
public function get($file) {
$query = $this->getQueryBuilder();
$query->selectFileCache();
$metadataQuery = $query->selectMetadata();
$metadataQuery = $query->selectMetadata($this->metadataManager);

if (is_string($file) || $file == '') {
// normalize file
Expand Down Expand Up @@ -236,7 +231,7 @@ public function getFolderContentsById($fileId) {
->whereParent($fileId)
->orderBy('name', 'ASC');

$metadataQuery = $query->selectMetadata();
$metadataQuery = $query->selectMetadata($this->metadataManager);

$result = $query->execute();
$files = $result->fetchAll();
Expand Down Expand Up @@ -305,7 +300,7 @@ public function insert($file, array $data) {
$values['storage'] = $storageId;

try {
$builder = $this->connection->getQueryBuilder();
$builder = $this->getQueryBuilder();
$builder->insert('filecache');

foreach ($values as $column => $value) {
Expand Down Expand Up @@ -333,9 +328,9 @@ public function insert($file, array $data) {
}
} catch (UniqueConstraintViolationException $e) {
// entry exists already
if ($this->connection->inTransaction()) {
$this->connection->commit();
$this->connection->beginTransaction();
if ($this->cacheDb->inTransaction($this->getNumericStorageId())) {
$this->cacheDb->commit($this->getNumericStorageId());
$this->cacheDb->beginTransaction($this->getNumericStorageId());
}
}

Expand Down Expand Up @@ -699,7 +694,7 @@ public function moveFromCache(ICache $sourceCache, $sourcePath, $targetPath) {
if ($sourceData['mimetype'] === 'httpd/unix-directory') {
//update all child entries
$sourceLength = mb_strlen($sourcePath);
$query = $this->connection->getQueryBuilder();
$query = $this->getQueryBuilder();

$fun = $query->func();
$newPathFunction = $fun->concat(
Expand All @@ -711,7 +706,7 @@ public function moveFromCache(ICache $sourceCache, $sourcePath, $targetPath) {
->set('path_hash', $fun->md5($newPathFunction))
->set('path', $newPathFunction)
->where($query->expr()->eq('storage', $query->createNamedParameter($sourceStorageId, IQueryBuilder::PARAM_INT)))
->andWhere($query->expr()->like('path', $query->createNamedParameter($this->connection->escapeLikeParameter($sourcePath) . '/%')));
->andWhere($query->expr()->like('path', $query->createNamedParameter($query->escapeLikeParameter($sourcePath) . '/%')));

// when moving from an encrypted storage to a non-encrypted storage remove the `encrypted` mark
if ($sourceCache->hasEncryptionWrapper() && !$this->hasEncryptionWrapper()) {
Expand All @@ -723,26 +718,26 @@ public function moveFromCache(ICache $sourceCache, $sourcePath, $targetPath) {
$retryLimit = 4;
for ($i = 1; $i <= $retryLimit; $i++) {
try {
$this->connection->beginTransaction();
$this->cacheDb->beginTransaction();

Check failure

Code scanning / Psalm

TooFewArguments

Too few arguments for OC\Files\Cache\CacheDatabase::beginTransaction - expecting storageId to be passed

Check failure

Code scanning / Psalm

TooFewArguments

Too few arguments for method OC\Files\Cache\CacheDatabase::begintransaction saw 0
$query->executeStatement();
break;
} catch (\OC\DatabaseException $e) {
$this->connection->rollBack();
$this->cacheDb->rollBack($this->getNumericStorageId());
throw $e;
} catch (RetryableException $e) {
// Simply throw if we already retried 4 times.
if ($i === $retryLimit) {
throw $e;
}

$this->connection->rollBack();
$this->cacheDb->rollBack();

Check failure

Code scanning / Psalm

TooFewArguments

Too few arguments for OC\Files\Cache\CacheDatabase::rollBack - expecting storageId to be passed

Check failure

Code scanning / Psalm

TooFewArguments

Too few arguments for method OC\Files\Cache\CacheDatabase::rollback saw 0

// Sleep a bit to give some time to the other transaction to finish.
usleep(100 * 1000 * $i);
}
}
} else {
$this->connection->beginTransaction();
$this->cacheDb->beginTransaction();

Check failure

Code scanning / Psalm

TooFewArguments

Too few arguments for OC\Files\Cache\CacheDatabase::beginTransaction - expecting storageId to be passed

Check failure

Code scanning / Psalm

TooFewArguments

Too few arguments for method OC\Files\Cache\CacheDatabase::begintransaction saw 0
}

$query = $this->getQueryBuilder();
Expand All @@ -761,7 +756,7 @@ public function moveFromCache(ICache $sourceCache, $sourcePath, $targetPath) {

$query->execute();

$this->connection->commit();
$this->cacheDb->commit($this->getNumericStorageId());

if ($sourceCache->getNumericStorageId() !== $this->getNumericStorageId()) {
$this->eventDispatcher->dispatchTyped(new CacheEntryRemovedEvent($this->storage, $sourcePath, $sourceId, $sourceCache->getNumericStorageId()));
Expand All @@ -787,7 +782,7 @@ public function clear() {
->whereStorageId($this->getNumericStorageId());
$query->execute();

$query = $this->connection->getQueryBuilder();
$query = $this->getQueryBuilder();
$query->delete('storages')
->where($query->expr()->eq('id', $query->createNamedParameter($this->storageId)));
$query->execute();
Expand Down Expand Up @@ -861,8 +856,8 @@ public function searchByMime($mimetype) {
return $this->searchQuery(new SearchQuery($operator, 0, 0, [], null));
}

public function searchQuery(ISearchQuery $searchQuery) {
return current($this->querySearchHelper->searchInCaches($searchQuery, [$this]));
public function searchQuery(ISearchQuery $query) {
return current($this->querySearchHelper->searchInCaches($query, [$this]));
}

/**
Expand Down Expand Up @@ -1099,27 +1094,7 @@ public function getPathById($id) {
* @deprecated use getPathById() instead
*/
public static function getById($id) {
$query = \OC::$server->getDatabaseConnection()->getQueryBuilder();
$query->select('path', 'storage')
->from('filecache')
->where($query->expr()->eq('fileid', $query->createNamedParameter($id, IQueryBuilder::PARAM_INT)));

$result = $query->execute();
$row = $result->fetch();
$result->closeCursor();

if ($row) {
$numericId = $row['storage'];
$path = $row['path'];
} else {
return null;
}

if ($id = Storage::getStorageId($numericId)) {
return [$id, $path];
} else {
return null;
}
throw new \Exception("Cache::getById has been removed");
}

/**
Expand Down
Loading