Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
8eaddbc
feat: track expected output columns in query builder
icewind1991 Jul 31, 2024
b1744e7
fix: don't make ICacheFactory depend on database
icewind1991 Aug 8, 2024
8f57d46
fix: delay calculating global cache prefix untill a cache is created
icewind1991 Aug 20, 2024
4ec53e7
feat: add option to automatically partition queries by specific tables
icewind1991 Jun 13, 2024
82d7eaf
feat: implement distributing partitioned queries over multiple shards
icewind1991 Jul 31, 2024
ecf1cc2
test: mark share test cleanup as running across all shards
icewind1991 Jul 16, 2024
3e51939
fix: only allow pre-defined shards
icewind1991 Jul 18, 2024
22f76fc
test: run sharding tests in ci
icewind1991 Jul 18, 2024
693ee5e
fix: hint storage id in more places
icewind1991 Jul 19, 2024
0e40fa4
fix: run mimetype repair query across all shards
icewind1991 Jul 19, 2024
1b6d76a
test: fix share provider tests for sharding
icewind1991 Jul 19, 2024
5500723
fix: make background scan job compatible with sharding
icewind1991 Jul 25, 2024
ddecae8
fix: adjust systemtag orphan cleanup query to work with sharding
icewind1991 Jul 31, 2024
dc5f0f5
fix: fix share cleanup for deleted groups with sharding
icewind1991 Aug 6, 2024
b264559
fix: implement sharding compatible cleanup for various bits
icewind1991 Aug 15, 2024
57ffbb7
fix: make preload custom proterties sharding compatible
icewind1991 Aug 21, 2024
e2bff39
fix: mark systemconfig value as not being tainted because they are im…
icewind1991 Aug 22, 2024
e5a8f99
chore: Apply php:cs recommendations
artonge Aug 28, 2024
140b36f
fix: Backport to 30
artonge Aug 28, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix: delay calculating global cache prefix untill a cache is created
Signed-off-by: Robin Appelman <[email protected]>
  • Loading branch information
icewind1991 authored and artonge committed Aug 28, 2024
commit 8f57d46a0bbe452bb153890a08d686b39daa5614
45 changes: 37 additions & 8 deletions lib/private/Memcache/Factory.php
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
class Factory implements ICacheFactory {
public const NULL_CACHE = NullCache::class;

private string $globalPrefix;
private ?string $globalPrefix = null;

private LoggerInterface $logger;

Expand All @@ -40,17 +40,23 @@ class Factory implements ICacheFactory {
private IProfiler $profiler;

/**
* @param string $globalPrefix
* @param callable $globalPrefixClosure
* @param LoggerInterface $logger
* @param ?class-string<ICache> $localCacheClass
* @param ?class-string<ICache> $distributedCacheClass
* @param ?class-string<IMemcache> $lockingCacheClass
* @param string $logFile
*/
public function __construct(string $globalPrefix, LoggerInterface $logger, IProfiler $profiler,
?string $localCacheClass = null, ?string $distributedCacheClass = null, ?string $lockingCacheClass = null, string $logFile = '') {
public function __construct(
private $globalPrefixClosure,
LoggerInterface $logger,
IProfiler $profiler,
?string $localCacheClass = null,
?string $distributedCacheClass = null,
?string $lockingCacheClass = null,
string $logFile = ''
) {
$this->logFile = $logFile;
$this->globalPrefix = $globalPrefix;

if (!$localCacheClass) {
$localCacheClass = self::NULL_CACHE;
Expand All @@ -59,6 +65,7 @@ public function __construct(string $globalPrefix, LoggerInterface $logger, IProf
if (!$distributedCacheClass) {
$distributedCacheClass = $localCacheClass;
}

$distributedCacheClass = ltrim($distributedCacheClass, '\\');

$missingCacheMessage = 'Memcache {class} not available for {use} cache';
Expand All @@ -85,15 +92,27 @@ public function __construct(string $globalPrefix, LoggerInterface $logger, IProf
$this->profiler = $profiler;
}

private function getGlobalPrefix(): ?string {
if (is_null($this->globalPrefix)) {
$this->globalPrefix = ($this->globalPrefixClosure)();
}
return $this->globalPrefix;
}

/**
* create a cache instance for storing locks
*
* @param string $prefix
* @return IMemcache
*/
public function createLocking(string $prefix = ''): IMemcache {
$globalPrefix = $this->getGlobalPrefix();
if (is_null($globalPrefix)) {
return new ArrayCache($prefix);
}

assert($this->lockingCacheClass !== null);
$cache = new $this->lockingCacheClass($this->globalPrefix . '/' . $prefix);
$cache = new $this->lockingCacheClass($globalPrefix . '/' . $prefix);
if ($this->lockingCacheClass === Redis::class && $this->profiler->isEnabled()) {
// We only support the profiler with Redis
$cache = new ProfilerWrapperCache($cache, 'Locking');
Expand All @@ -114,8 +133,13 @@ public function createLocking(string $prefix = ''): IMemcache {
* @return ICache
*/
public function createDistributed(string $prefix = ''): ICache {
$globalPrefix = $this->getGlobalPrefix();
if (is_null($globalPrefix)) {
return new ArrayCache($prefix);
}

assert($this->distributedCacheClass !== null);
$cache = new $this->distributedCacheClass($this->globalPrefix . '/' . $prefix);
$cache = new $this->distributedCacheClass($globalPrefix . '/' . $prefix);
if ($this->distributedCacheClass === Redis::class && $this->profiler->isEnabled()) {
// We only support the profiler with Redis
$cache = new ProfilerWrapperCache($cache, 'Distributed');
Expand All @@ -136,8 +160,13 @@ public function createDistributed(string $prefix = ''): ICache {
* @return ICache
*/
public function createLocal(string $prefix = ''): ICache {
$globalPrefix = $this->getGlobalPrefix();
if (is_null($globalPrefix)) {
return new ArrayCache($prefix);
}

assert($this->localCacheClass !== null);
$cache = new $this->localCacheClass($this->globalPrefix . '/' . $prefix);
$cache = new $this->localCacheClass($globalPrefix . '/' . $prefix);
if ($this->localCacheClass === Redis::class && $this->profiler->isEnabled()) {
// We only support the profiler with Redis
$cache = new ProfilerWrapperCache($cache, 'Local');
Expand Down
51 changes: 29 additions & 22 deletions lib/private/Server.php
Original file line number Diff line number Diff line change
Expand Up @@ -637,7 +637,7 @@ public function __construct($webRoot, \OC\Config $config) {

$this->registerService(Factory::class, function (Server $c) {
$profiler = $c->get(IProfiler::class);
$arrayCacheFactory = new \OC\Memcache\Factory('', $c->get(LoggerInterface::class),
$arrayCacheFactory = new \OC\Memcache\Factory(fn () => '', $c->get(LoggerInterface::class),
$profiler,
ArrayCache::class,
ArrayCache::class,
Expand All @@ -647,33 +647,40 @@ public function __construct($webRoot, \OC\Config $config) {
$config = $c->get(SystemConfig::class);

if ($config->getValue('installed', false) && !(defined('PHPUNIT_RUN') && PHPUNIT_RUN)) {
if (!$config->getValue('log_query')) {
try {
$v = \OC_App::getAppVersions();
} catch (\Doctrine\DBAL\Exception $e) {
// Database service probably unavailable
// Probably related to https://github.com/nextcloud/server/issues/37424
return $arrayCacheFactory;
$logQuery = $config->getValue('log_query');
$prefixClosure = function () use ($logQuery) {
if (!$logQuery) {
try {
$v = \OC_App::getAppVersions();
} catch (\Doctrine\DBAL\Exception $e) {
// Database service probably unavailable
// Probably related to https://github.com/nextcloud/server/issues/37424
return null;
}
} else {
// If the log_query is enabled, we can not get the app versions
// as that does a query, which will be logged and the logging
// depends on redis and here we are back again in the same function.
$v = [
'log_query' => 'enabled',
];
}
} else {
// If the log_query is enabled, we can not get the app versions
// as that does a query, which will be logged and the logging
// depends on redis and here we are back again in the same function.
$v = [
'log_query' => 'enabled',
];
}
$v['core'] = implode(',', \OC_Util::getVersion());
$version = implode(',', $v);
$instanceId = \OC_Util::getInstanceId();
$path = \OC::$SERVERROOT;
$prefix = md5($instanceId . '-' . $version . '-' . $path);
return new \OC\Memcache\Factory($prefix,
$v['core'] = implode(',', \OC_Util::getVersion());
$version = implode(',', $v);
$instanceId = \OC_Util::getInstanceId();
$path = \OC::$SERVERROOT;
return md5($instanceId . '-' . $version . '-' . $path);
};
return new \OC\Memcache\Factory($prefixClosure,
$c->get(LoggerInterface::class),
$profiler,
/** @psalm-taint-escape callable */
$config->getValue('memcache.local', null),
/** @psalm-taint-escape callable */
$config->getValue('memcache.distributed', null),
/** @psalm-taint-escape callable */
$config->getValue('memcache.locking', null),
/** @psalm-taint-escape callable */
$config->getValue('redis_log_file')
);
}
Expand Down
6 changes: 3 additions & 3 deletions tests/lib/Memcache/FactoryTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ public function testCacheAvailability($localCache, $distributedCache, $lockingCa
$expectedLocalCache, $expectedDistributedCache, $expectedLockingCache) {
$logger = $this->getMockBuilder(LoggerInterface::class)->getMock();
$profiler = $this->getMockBuilder(IProfiler::class)->getMock();
$factory = new \OC\Memcache\Factory('abc', $logger, $profiler, $localCache, $distributedCache, $lockingCache);
$factory = new \OC\Memcache\Factory(fn () => 'abc', $logger, $profiler, $localCache, $distributedCache, $lockingCache);
$this->assertTrue(is_a($factory->createLocal(), $expectedLocalCache));
$this->assertTrue(is_a($factory->createDistributed(), $expectedDistributedCache));
$this->assertTrue(is_a($factory->createLocking(), $expectedLockingCache));
Expand All @@ -124,13 +124,13 @@ public function testCacheNotAvailableException($localCache, $distributedCache) {

$logger = $this->getMockBuilder(LoggerInterface::class)->getMock();
$profiler = $this->getMockBuilder(IProfiler::class)->getMock();
new \OC\Memcache\Factory('abc', $logger, $profiler, $localCache, $distributedCache);
new \OC\Memcache\Factory(fn () => 'abc', $logger, $profiler, $localCache, $distributedCache);
}

public function testCreateInMemory(): void {
$logger = $this->getMockBuilder(LoggerInterface::class)->getMock();
$profiler = $this->getMockBuilder(IProfiler::class)->getMock();
$factory = new \OC\Memcache\Factory('abc', $logger, $profiler, null, null, null);
$factory = new \OC\Memcache\Factory(fn () => 'abc', $logger, $profiler, null, null, null);

$cache = $factory->createInMemory();
$cache->set('test', 48);
Expand Down