From c9b055a0d06065ab37493113a0f7276b50168114 Mon Sep 17 00:00:00 2001 From: Benjamin Gaussorgues Date: Mon, 13 Oct 2025 17:41:45 +0200 Subject: [PATCH 1/2] feat(database): introduce Snowflake IDs generator Signed-off-by: Benjamin Gaussorgues --- .github/workflows/phpunit-32bits.yml | 12 +- apps/settings/lib/SetupChecks/PhpModules.php | 1 + composer.json | 5 +- core/Command/SnowflakeDecodeId.php | 48 +++++++ core/register_command.php | 2 + lib/composer/composer/autoload_classmap.php | 5 + lib/composer/composer/autoload_static.php | 5 + lib/composer/composer/platform_check.php | 4 +- lib/private/Server.php | 7 + lib/private/Snowflake/Decoder.php | 122 ++++++++++++++++ lib/private/Snowflake/Generator.php | 138 +++++++++++++++++++ lib/public/Snowflake/IDecoder.php | 35 +++++ lib/public/Snowflake/IGenerator.php | 46 +++++++ tests/lib/Snowflake/DecoderTest.php | 69 ++++++++++ tests/lib/Snowflake/GeneratorTest.php | 80 +++++++++++ 15 files changed, 570 insertions(+), 9 deletions(-) create mode 100644 core/Command/SnowflakeDecodeId.php create mode 100644 lib/private/Snowflake/Decoder.php create mode 100644 lib/private/Snowflake/Generator.php create mode 100644 lib/public/Snowflake/IDecoder.php create mode 100644 lib/public/Snowflake/IGenerator.php create mode 100644 tests/lib/Snowflake/DecoderTest.php create mode 100644 tests/lib/Snowflake/GeneratorTest.php diff --git a/.github/workflows/phpunit-32bits.yml b/.github/workflows/phpunit-32bits.yml index 3a0ae82678690..81c2b62da692f 100644 --- a/.github/workflows/phpunit-32bits.yml +++ b/.github/workflows/phpunit-32bits.yml @@ -5,9 +5,10 @@ name: PHPUnit 32bits on: pull_request: paths: - - 'version.php' - - '.github/workflows/phpunit-32bits.yml' - - 'tests/phpunit-autotest.xml' + - "version.php" + - ".github/workflows/phpunit-32bits.yml" + - "tests/phpunit-autotest.xml" + - "lib/private/Snowflake/*" workflow_dispatch: schedule: - cron: "15 1 * * 1-6" @@ -30,7 +31,7 @@ jobs: strategy: fail-fast: false matrix: - php-versions: ['8.2', '8.3', '8.4'] + php-versions: ["8.2", "8.3", "8.4"] steps: - name: Checkout server @@ -51,8 +52,7 @@ jobs: extensions: bz2, ctype, curl, dom, fileinfo, gd, iconv, imagick, intl, json, libxml, mbstring, openssl, pcntl, posix, redis, session, simplexml, xmlreader, xmlwriter, zip, zlib, sqlite, pdo_sqlite, apcu, ldap coverage: none ini-file: development - ini-values: - apc.enabled=on, apc.enable_cli=on, disable_functions= # https://github.com/shivammathur/setup-php/discussions/573 + ini-values: apc.enabled=on, apc.enable_cli=on, disable_functions= # https://github.com/shivammathur/setup-php/discussions/573 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/apps/settings/lib/SetupChecks/PhpModules.php b/apps/settings/lib/SetupChecks/PhpModules.php index a83cc717a5feb..49bf6b2ff9366 100644 --- a/apps/settings/lib/SetupChecks/PhpModules.php +++ b/apps/settings/lib/SetupChecks/PhpModules.php @@ -31,6 +31,7 @@ class PhpModules implements ISetupCheck { 'zlib', ]; protected const RECOMMENDED_MODULES = [ + 'apcu', 'exif', 'gmp', 'intl', diff --git a/composer.json b/composer.json index 8b4ace64b80c3..fdcadc7a12154 100644 --- a/composer.json +++ b/composer.json @@ -11,7 +11,9 @@ } }, "autoload": { - "exclude-from-classmap": ["**/bamarni/composer-bin-plugin/**"], + "exclude-from-classmap": [ + "**/bamarni/composer-bin-plugin/**" + ], "files": [ "lib/public/Log/functions.php" ], @@ -25,6 +27,7 @@ }, "require": { "php": "^8.2", + "ext-apcu": "*", "ext-ctype": "*", "ext-curl": "*", "ext-dom": "*", diff --git a/core/Command/SnowflakeDecodeId.php b/core/Command/SnowflakeDecodeId.php new file mode 100644 index 0000000000000..3626046247ba6 --- /dev/null +++ b/core/Command/SnowflakeDecodeId.php @@ -0,0 +1,48 @@ +setName('decode-snowflake') + ->setDescription('Decode Snowflake IDs used by Nextcloud') + ->addArgument('snowflake-id', InputArgument::REQUIRED, 'Nextcloud Snowflake ID to decode'); + } + + protected function execute(InputInterface $input, OutputInterface $output): int { + $snowflakeId = $input->getArgument('snowflake-id'); + $data = (new Decoder)->decode($snowflakeId); + + $rows = [ + ['Snowflake ID', $snowflakeId], + ['Seconds', $data['seconds']], + ['Milliseconds', $data['milliseconds']], + ['Created from CLI', $data['isCli'] ? 'yes' : 'no'], + ['Server ID', $data['serverId']], + ['Sequence ID', $data['sequenceId']], + ['Creation timestamp', $data['createdAt']->format('U.v')], + ['Creation date', $data['createdAt']->format('Y-m-d H:i:s.v')], + ]; + + $table = new Table($output); + $table->setRows($rows); + $table->render(); + + return Base::SUCCESS; + } +} diff --git a/core/register_command.php b/core/register_command.php index f6c0b9466b5cb..58aed05ba68a6 100644 --- a/core/register_command.php +++ b/core/register_command.php @@ -85,6 +85,7 @@ use OC\Core\Command\Security\ListCertificates; use OC\Core\Command\Security\RemoveCertificate; use OC\Core\Command\SetupChecks; +use OC\Core\Command\SnowflakeDecodeId; use OC\Core\Command\Status; use OC\Core\Command\SystemTag\Edit; use OC\Core\Command\TaskProcessing\EnabledCommand; @@ -246,6 +247,7 @@ $application->add(Server::get(BruteforceAttempts::class)); $application->add(Server::get(BruteforceResetAttempts::class)); $application->add(Server::get(SetupChecks::class)); + $application->add(Server::get(SnowflakeDecodeId::class)); $application->add(Server::get(Get::class)); $application->add(Server::get(GetCommand::class)); diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index 3e57c603ca605..d14d80b23f76c 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -822,6 +822,8 @@ 'OCP\\Share_Backend' => $baseDir . '/lib/public/Share_Backend.php', 'OCP\\Share_Backend_Collection' => $baseDir . '/lib/public/Share_Backend_Collection.php', 'OCP\\Share_Backend_File_Dependent' => $baseDir . '/lib/public/Share_Backend_File_Dependent.php', + 'OCP\\Snowflake\\IDecoder' => $baseDir . '/lib/public/Snowflake/IDecoder.php', + 'OCP\\Snowflake\\IGenerator' => $baseDir . '/lib/public/Snowflake/IGenerator.php', 'OCP\\SpeechToText\\Events\\AbstractTranscriptionEvent' => $baseDir . '/lib/public/SpeechToText/Events/AbstractTranscriptionEvent.php', 'OCP\\SpeechToText\\Events\\TranscriptionFailedEvent' => $baseDir . '/lib/public/SpeechToText/Events/TranscriptionFailedEvent.php', 'OCP\\SpeechToText\\Events\\TranscriptionSuccessfulEvent' => $baseDir . '/lib/public/SpeechToText/Events/TranscriptionSuccessfulEvent.php', @@ -1352,6 +1354,7 @@ 'OC\\Core\\Command\\Security\\ListCertificates' => $baseDir . '/core/Command/Security/ListCertificates.php', 'OC\\Core\\Command\\Security\\RemoveCertificate' => $baseDir . '/core/Command/Security/RemoveCertificate.php', 'OC\\Core\\Command\\SetupChecks' => $baseDir . '/core/Command/SetupChecks.php', + 'OC\\Core\\Command\\SnowflakeDecodeId' => $baseDir . '/core/Command/SnowflakeDecodeId.php', 'OC\\Core\\Command\\Status' => $baseDir . '/core/Command/Status.php', 'OC\\Core\\Command\\SystemTag\\Add' => $baseDir . '/core/Command/SystemTag/Add.php', 'OC\\Core\\Command\\SystemTag\\Delete' => $baseDir . '/core/Command/SystemTag/Delete.php', @@ -2103,6 +2106,8 @@ 'OC\\Share\\Constants' => $baseDir . '/lib/private/Share/Constants.php', 'OC\\Share\\Helper' => $baseDir . '/lib/private/Share/Helper.php', 'OC\\Share\\Share' => $baseDir . '/lib/private/Share/Share.php', + 'OC\\Snowflake\\Decoder' => $baseDir . '/lib/private/Snowflake/Decoder.php', + 'OC\\Snowflake\\Generator' => $baseDir . '/lib/private/Snowflake/Generator.php', 'OC\\SpeechToText\\SpeechToTextManager' => $baseDir . '/lib/private/SpeechToText/SpeechToTextManager.php', 'OC\\SpeechToText\\TranscriptionJob' => $baseDir . '/lib/private/SpeechToText/TranscriptionJob.php', 'OC\\StreamImage' => $baseDir . '/lib/private/StreamImage.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index e54ff53b60988..fd607dfc14a06 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -863,6 +863,8 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OCP\\Share_Backend' => __DIR__ . '/../../..' . '/lib/public/Share_Backend.php', 'OCP\\Share_Backend_Collection' => __DIR__ . '/../../..' . '/lib/public/Share_Backend_Collection.php', 'OCP\\Share_Backend_File_Dependent' => __DIR__ . '/../../..' . '/lib/public/Share_Backend_File_Dependent.php', + 'OCP\\Snowflake\\IDecoder' => __DIR__ . '/../../..' . '/lib/public/Snowflake/IDecoder.php', + 'OCP\\Snowflake\\IGenerator' => __DIR__ . '/../../..' . '/lib/public/Snowflake/IGenerator.php', 'OCP\\SpeechToText\\Events\\AbstractTranscriptionEvent' => __DIR__ . '/../../..' . '/lib/public/SpeechToText/Events/AbstractTranscriptionEvent.php', 'OCP\\SpeechToText\\Events\\TranscriptionFailedEvent' => __DIR__ . '/../../..' . '/lib/public/SpeechToText/Events/TranscriptionFailedEvent.php', 'OCP\\SpeechToText\\Events\\TranscriptionSuccessfulEvent' => __DIR__ . '/../../..' . '/lib/public/SpeechToText/Events/TranscriptionSuccessfulEvent.php', @@ -1393,6 +1395,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Core\\Command\\Security\\ListCertificates' => __DIR__ . '/../../..' . '/core/Command/Security/ListCertificates.php', 'OC\\Core\\Command\\Security\\RemoveCertificate' => __DIR__ . '/../../..' . '/core/Command/Security/RemoveCertificate.php', 'OC\\Core\\Command\\SetupChecks' => __DIR__ . '/../../..' . '/core/Command/SetupChecks.php', + 'OC\\Core\\Command\\SnowflakeDecodeId' => __DIR__ . '/../../..' . '/core/Command/SnowflakeDecodeId.php', 'OC\\Core\\Command\\Status' => __DIR__ . '/../../..' . '/core/Command/Status.php', 'OC\\Core\\Command\\SystemTag\\Add' => __DIR__ . '/../../..' . '/core/Command/SystemTag/Add.php', 'OC\\Core\\Command\\SystemTag\\Delete' => __DIR__ . '/../../..' . '/core/Command/SystemTag/Delete.php', @@ -2144,6 +2147,8 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Share\\Constants' => __DIR__ . '/../../..' . '/lib/private/Share/Constants.php', 'OC\\Share\\Helper' => __DIR__ . '/../../..' . '/lib/private/Share/Helper.php', 'OC\\Share\\Share' => __DIR__ . '/../../..' . '/lib/private/Share/Share.php', + 'OC\\Snowflake\\Decoder' => __DIR__ . '/../../..' . '/lib/private/Snowflake/Decoder.php', + 'OC\\Snowflake\\Generator' => __DIR__ . '/../../..' . '/lib/private/Snowflake/Generator.php', 'OC\\SpeechToText\\SpeechToTextManager' => __DIR__ . '/../../..' . '/lib/private/SpeechToText/SpeechToTextManager.php', 'OC\\SpeechToText\\TranscriptionJob' => __DIR__ . '/../../..' . '/lib/private/SpeechToText/TranscriptionJob.php', 'OC\\StreamImage' => __DIR__ . '/../../..' . '/lib/private/StreamImage.php', diff --git a/lib/composer/composer/platform_check.php b/lib/composer/composer/platform_check.php index 2beb149183886..14bf88da3a275 100644 --- a/lib/composer/composer/platform_check.php +++ b/lib/composer/composer/platform_check.php @@ -4,8 +4,8 @@ $issues = array(); -if (!(PHP_VERSION_ID >= 80100)) { - $issues[] = 'Your Composer dependencies require a PHP version ">= 8.1.0". You are running ' . PHP_VERSION . '.'; +if (!(PHP_VERSION_ID >= 80200)) { + $issues[] = 'Your Composer dependencies require a PHP version ">= 8.2.0". You are running ' . PHP_VERSION . '.'; } if ($issues) { diff --git a/lib/private/Server.php b/lib/private/Server.php index 927d2ce322422..01b6ec0e22b69 100644 --- a/lib/private/Server.php +++ b/lib/private/Server.php @@ -114,6 +114,8 @@ use OC\SetupCheck\SetupCheckManager; use OC\Share20\ProviderFactory; use OC\Share20\ShareHelper; +use OC\Snowflake\Decoder; +use OC\Snowflake\Generator; use OC\SpeechToText\SpeechToTextManager; use OC\SystemTag\ManagerFactory as SystemTagManagerFactory; use OC\Talk\Broker; @@ -222,6 +224,8 @@ use OCP\SetupCheck\ISetupCheckManager; use OCP\Share\IProviderFactory; use OCP\Share\IShareHelper; +use OCP\Snowflake\IDecoder; +use OCP\Snowflake\IGenerator; use OCP\SpeechToText\ISpeechToTextManager; use OCP\SystemTag\ISystemTagManager; use OCP\SystemTag\ISystemTagObjectMapper; @@ -1245,6 +1249,9 @@ public function __construct($webRoot, \OC\Config $config) { $this->registerAlias(ISignatureManager::class, SignatureManager::class); + $this->registerAlias(IGenerator::class, Generator::class); + $this->registerAlias(IDecoder::class, Decoder::class); + $this->connectDispatcher(); } diff --git a/lib/private/Snowflake/Decoder.php b/lib/private/Snowflake/Decoder.php new file mode 100644 index 0000000000000..e4ec72e50d2c0 --- /dev/null +++ b/lib/private/Snowflake/Decoder.php @@ -0,0 +1,122 @@ +, serverId: int<0, 1023>, sequenceId: int<0,4095>, isCli: bool} $data */ + $data = PHP_INT_SIZE === 8 + ? $this->decode64bits((int)$snowflakeId) + : $this->decode32bits($snowflakeId); + + $data['createdAt'] = new \DateTimeImmutable( + sprintf( + '@%d.%03d', + $data['seconds'] + IGenerator::TS_OFFSET + intdiv($data['milliseconds'], 1000), + $data['milliseconds'] % 1000, + ) + ); + + return $data; + } + + private function decode64bits(int $snowflakeId): array { + $firstHalf = $snowflakeId >> 32; + $secondHalf = $snowflakeId & 0xFFFFFFFF; + + $seconds = $firstHalf & 0x7FFFFFFF; + $milliseconds = $secondHalf >> 22; + + return [ + 'seconds' => $seconds, + 'milliseconds' => $milliseconds, + 'serverId' => ($secondHalf >> 13) & 0x1FF, + 'sequenceId' => $secondHalf & 0xFFF, + 'isCli' => (bool)(($secondHalf >> 12) & 0x1), + ]; + } + + private function decode32bits(string $snowflakeId): array { + $id = $this->convertBase16($snowflakeId); + + $firstQuarter = (int)hexdec(substr($id, 0, 4)); + $secondQuarter = (int)hexdec(substr($id, 4, 4)); + $thirdQuarter = (int)hexdec(substr($id, 8, 4)); + $fourthQuarter = (int)hexdec(substr($id, 12, 4)); + + $seconds = (($firstQuarter & 0x7FFF) << 16) | ($secondQuarter & 0xFFFF); + $milliseconds = ($thirdQuarter >> 6) & 0x3FF; + + return [ + 'seconds' => $seconds, + 'milliseconds' => $milliseconds, + 'serverId' => (($thirdQuarter & 0x3F) << 3) | (($fourthQuarter >> 13) & 0x7), + 'sequenceId' => $fourthQuarter & 0xFFF, + 'isCli' => (bool)(($fourthQuarter >> 12) & 0x1), + ]; + } + + /** + * Convert base 10 number to base 16, padded to 16 characters + * + * Required on 32 bits systems as base_convert will lose precision with large numbers + */ + private function convertBase16(string $decimal): string { + $hex = ''; + $digits = '0123456789ABCDEF'; + + while (strlen($decimal) > 0 && $decimal !== '0') { + $remainder = 0; + $newDecimal = ''; + + // Perform division by 16 manually for arbitrary precision + for ($i = 0; $i < strlen($decimal); $i++) { + $digit = (int)$decimal[$i]; + $current = $remainder * 10 + $digit; + + if ($current >= 16) { + $quotient = (int)($current / 16); + $remainder = $current % 16; + $newDecimal .= chr(ord('0') + $quotient); + } else { + $remainder = $current; + // Only add quotient digit if we already have some digits in result + if (strlen($newDecimal) > 0) { + $newDecimal .= '0'; + } + } + } + + // Add the remainder (0-15) as hex digit + $hex = $digits[$remainder] . $hex; + + // Update decimal for next iteration + $decimal = ltrim($newDecimal, '0'); + } + + return str_pad($hex, 16, '0', STR_PAD_LEFT); + } +} diff --git a/lib/private/Snowflake/Generator.php b/lib/private/Snowflake/Generator.php new file mode 100644 index 0000000000000..f378482a315dd --- /dev/null +++ b/lib/private/Snowflake/Generator.php @@ -0,0 +1,138 @@ +getCurrentTime(); + + $serverId = $this->getServerId() & 0x1FF; // Keep 9 bits + $isCli = (int)$this->isCli(); // 1 bit + $sequenceId = $this->getSequenceId($seconds, $milliseconds, $serverId); // 12 bits + if ($sequenceId > 0xFFF || $sequenceId === false) { + // Throttle a bit, wait for next millisecond + usleep(1000); + return $this->nextId(); + } + + if (PHP_INT_SIZE === 8) { + $firstHalf = $seconds & 0x7FFFFFFF; + $secondHalf = (($milliseconds & 0x3FF) << 22) | ($serverId << 13) | ($isCli << 12) | $sequenceId; + return (string)($firstHalf << 32 | $secondHalf); + } + + // Fallback for 32 bits systems + $firstQuarter = ($seconds >> 16) & 0x7FFF; + $secondQuarter = $seconds & 0xFFFF; + $thirdQuarter = ($milliseconds & 0x3FF) << 6 | ($serverId >> 3) & 0x3F; + $fourthQuarter = ($serverId & 0x7) << 13 | ($isCli & 0x1) << 12 | $sequenceId & 0xFFF; + + $bin = pack('n*', $firstQuarter, $secondQuarter, $thirdQuarter, $fourthQuarter); + + $bytes = unpack('C*', $bin); + if ($bytes === false) { + throw new \Exception('Fail to unpack'); + } + + return $this->convertToDecimal(array_values($bytes)); + } + + /** + * Mostly copied from Symfony: + * https://github.com/symfony/symfony/blob/v7.3.4/src/Symfony/Component/Uid/BinaryUtil.php#L49 + */ + private function convertToDecimal(array $bytes): string { + $base = 10; + $digits = ''; + + while ($count = \count($bytes)) { + $quotient = []; + $remainder = 0; + + for ($i = 0; $i !== $count; ++$i) { + $carry = $bytes[$i] + ($remainder << (PHP_INT_SIZE === 8 ? 16 : 8)); + $digit = intdiv($carry, $base); + $remainder = $carry % $base; + + if ($digit || $quotient) { + $quotient[] = $digit; + } + } + + $digits = $remainder . $digits; + $bytes = $quotient; + } + + return $digits; + } + + private function getCurrentTime(): array { + $time = $this->timeFactory->now(); + return [ + $time->getTimestamp() - self::TS_OFFSET, + (int)$time->format('v'), + ]; + } + + private function getServerId(): int { + return crc32(gethostname() ?: random_bytes(8)); + } + + private function isCli(): bool { + return PHP_SAPI === 'cli'; + } + + /** + * Generates sequence ID from APCu (general case) or random if APCu disabled or CLI + * + * @return int|false Sequence ID or false if APCu not ready + * @throws \Exception if there is an error with APCu + */ + private function getSequenceId(int $seconds, int $milliseconds, int $serverId): int|false { + $key = 'seq:' . $seconds . ':' . $milliseconds; + + // Use APCu as fastest local cache, but not shared between processes in CLI + if (!$this->isCli() && function_exists('apcu_enabled') && apcu_enabled()) { + if ((int)apcu_cache_info(true)['creation_time'] === $seconds) { + // APCu cache was just started + // It means a sequence was maybe deleted + return false; + } + + $sequenceId = apcu_inc($key, success: $success, ttl: 1); + if ($success === true) { + return $sequenceId; + } + + throw new \Exception('Failed to generate SnowflakeId with APCu'); + } + + // Otherwise, just return a random number + return random_int(0, 0xFFF - 1); + } +} diff --git a/lib/public/Snowflake/IDecoder.php b/lib/public/Snowflake/IDecoder.php new file mode 100644 index 0000000000000..f75019981b10e --- /dev/null +++ b/lib/public/Snowflake/IDecoder.php @@ -0,0 +1,35 @@ +, sequenceId: int<0,4095>, isCli: bool, seconds: positive-int, milliseconds: int<0,999>} + * @since 33.0 + */ + public function decode(string $snowflakeId): array; +} diff --git a/lib/public/Snowflake/IGenerator.php b/lib/public/Snowflake/IGenerator.php new file mode 100644 index 0000000000000..2a465861f8dbd --- /dev/null +++ b/lib/public/Snowflake/IGenerator.php @@ -0,0 +1,46 @@ +decoder = new Decoder(); + } + + #[DataProvider('provideSnowflakeIds')] + public function testDecode( + string $snowflakeId, + float $timestamp, + int $serverId, + int $sequenceId, + bool $isCli, + ): void { + $data = $this->decoder->decode($snowflakeId); + + $this->assertEquals($timestamp, (float)$data['createdAt']->format('U.v')); + $this->assertEquals($serverId, $data['serverId']); + $this->assertEquals($sequenceId, $data['sequenceId']); + $this->assertEquals($isCli, $data['isCli']); + } + + public static function provideSnowflakeIds(): array { + $data = [ + ['4688076898113587', 1760368327.984, 392, 2099, true], + // Max milliseconds + ['4190109696', 1759276800.999, 0, 0, false], + // Max serverId + ['4186112', 1759276800.0, 511, 0, false], + // Max sequenceId + ['4095', 1759276800.0, 0, 4095, false], + // Max isCli + ['4096', 1759276800.0, 0, 0, true], + // Min + ['0', 1759276800, 0, 0, false], + // Other + ['250159983611680096', 1817521710, 392, 1376, true], + ]; + + // 32 bits can't handle large timestamps correctly + if (PHP_INT_SIZE === 8) { + // Max all (can't happen because ms are up to 999) + $data[] = ['9223372036854775807', 3906760448.023, 511, 4095, true]; + // Max all (real) + $data[] = ['9223372036754112511', 3906760447.999, 511, 4095, true]; + // Max seconds + $data[] = ['9223372032559808512', 3906760447, 0, 0, false]; + } + + return $data; + } +} diff --git a/tests/lib/Snowflake/GeneratorTest.php b/tests/lib/Snowflake/GeneratorTest.php new file mode 100644 index 0000000000000..748d0f190422a --- /dev/null +++ b/tests/lib/Snowflake/GeneratorTest.php @@ -0,0 +1,80 @@ +decoder = new Decoder(); + } + public function testGenerator(): void { + $generator = new Generator(new TimeFactory()); + $snowflakeId = $generator->nextId(); + $data = $this->decoder->decode($generator->nextId()); + + $this->assertIsString($snowflakeId); + // Check timestamp + $this->assertGreaterThan(time() - 30, $data['createdAt']->format('U')); + + // Check serverId + $this->assertGreaterThanOrEqual(0, $data['serverId']); + $this->assertLessThanOrEqual(1023, $data['serverId']); + + // Check sequenceId + $this->assertGreaterThanOrEqual(0, $data['sequenceId']); + $this->assertLessThanOrEqual(4095, $data['sequenceId']); + + // Check CLI + $this->assertTrue($data['isCli']); + } + + #[DataProvider('provideSnowflakeData')] + public function testGeneratorWithFixedTime(string $date, int $expectedSeconds, int $expectedMilliseconds): void { + $dt = new \DateTimeImmutable($date); + $timeFactory = $this->createMock(ITimeFactory::class); + $timeFactory->method('now')->willReturn($dt); + + $generator = new Generator($timeFactory); + $data = $this->decoder->decode($generator->nextId()); + + $this->assertEquals($expectedSeconds, ($data['createdAt']->format('U') - IGenerator::TS_OFFSET)); + $this->assertEquals($expectedMilliseconds, (int)$data['createdAt']->format('v')); + } + + public static function provideSnowflakeData(): array { + $tests = [ + ['2025-10-01 00:00:00.000000', 0, 0], + ['2025-10-01 00:00:01.000000', 1, 0], + ['2025-10-01 00:00:00.001000', 0, 1], + ['2027-08-06 03:08:30.000975', 58244910, 0], + ['2030-06-21 12:59:33.100875', 149000373, 100], + ['2038-01-18 13:33:37.666666', 388157617, 666], + ]; + // Timestamp in 32 bits can't go after 2038. Add few cases for 64 bits. + if (PHP_INT_SIZE === 8) { + $tests[] = ['2039-12-31 23:59:59.999999', 449711999, 999]; + $tests[] = ['2086-06-21 12:59:33.010875', 1916225973, 10]; + } + + return $tests; + } +} From 336cc3fa3573e07585b25619e12621936ce11627 Mon Sep 17 00:00:00 2001 From: Carl Schwan Date: Mon, 13 Oct 2025 09:52:04 +0200 Subject: [PATCH 2/2] feat(Db): Use SnowflakeId for previews Allow to get an id for the storing the preview on disk before inserting the preview on the DB. Signed-off-by: Carl Schwan --- core/BackgroundJobs/MovePreviewJob.php | 3 + .../Version33000Date20251023110529.php | 44 ++++++++++ .../Version33000Date20251023120529.php | 86 +++++++++++++++++++ lib/composer/composer/autoload_classmap.php | 2 + lib/composer/composer/autoload_static.php | 2 + lib/private/AppFramework/Http/Dispatcher.php | 2 +- lib/private/DB/SchemaWrapper.php | 17 ++++ lib/private/Preview/Db/Preview.php | 15 ++-- lib/private/Preview/Db/PreviewMapper.php | 47 ++++++++-- lib/private/Preview/Generator.php | 36 ++++---- .../Preview/Storage/LocalPreviewStorage.php | 3 + lib/private/PreviewManager.php | 2 + lib/public/AppFramework/Db/Entity.php | 6 +- lib/public/DB/ISchemaWrapper.php | 7 ++ tests/lib/Preview/GeneratorTest.php | 4 + tests/lib/Preview/MovePreviewJobTest.php | 4 + tests/lib/Preview/PreviewMapperTest.php | 16 +++- tests/lib/Preview/PreviewServiceTest.php | 5 +- version.php | 2 +- 19 files changed, 260 insertions(+), 43 deletions(-) create mode 100644 core/Migrations/Version33000Date20251023110529.php create mode 100644 core/Migrations/Version33000Date20251023120529.php diff --git a/core/BackgroundJobs/MovePreviewJob.php b/core/BackgroundJobs/MovePreviewJob.php index 930f998a28bed..e0a9d5b5517c8 100644 --- a/core/BackgroundJobs/MovePreviewJob.php +++ b/core/BackgroundJobs/MovePreviewJob.php @@ -26,6 +26,7 @@ use OCP\IAppConfig; use OCP\IConfig; use OCP\IDBConnection; +use OCP\Snowflake\IGenerator; use Override; use Psr\Log\LoggerInterface; @@ -44,6 +45,7 @@ public function __construct( private readonly IMimeTypeDetector $mimeTypeDetector, private readonly IMimeTypeLoader $mimeTypeLoader, private readonly LoggerInterface $logger, + private readonly IGenerator $generator, IAppDataFactory $appDataFactory, ) { parent::__construct($time); @@ -136,6 +138,7 @@ private function processPreviews(int $fileId, bool $flatPath): void { $path = $fileId . '/' . $previewFile->getName(); /** @var SimpleFile $previewFile */ $preview = Preview::fromPath($path, $this->mimeTypeDetector); + $preview->setId($this->generator->nextId()); if (!$preview) { $this->logger->error('Unable to import old preview at path.'); continue; diff --git a/core/Migrations/Version33000Date20251023110529.php b/core/Migrations/Version33000Date20251023110529.php new file mode 100644 index 0000000000000..d87e8bc1f0b64 --- /dev/null +++ b/core/Migrations/Version33000Date20251023110529.php @@ -0,0 +1,44 @@ +hasTable('preview_locations')) { + $schema->dropAutoincrementColumn('preview_locations', 'id'); + } + + if ($schema->hasTable('preview_versions')) { + $schema->dropAutoincrementColumn('preview_versions', 'id'); + } + + if ($schema->hasTable('previews')) { + $schema->dropAutoincrementColumn('previews', 'id'); + } + + return $schema; + } +} diff --git a/core/Migrations/Version33000Date20251023120529.php b/core/Migrations/Version33000Date20251023120529.php new file mode 100644 index 0000000000000..bcc6d691a7a44 --- /dev/null +++ b/core/Migrations/Version33000Date20251023120529.php @@ -0,0 +1,86 @@ +hasTable('preview_locations')) { + $table = $schema->getTable('preview_locations'); + $table->addUniqueIndex(['bucket_name', 'object_store_name'], 'unique_bucket_store'); + } + + return $schema; + } + + public function preSchemaChange(IOutput $output, \Closure $schemaClosure, array $options) { + // This shouldn't run on a production instance, only daily + $qb = $this->connection->getQueryBuilder(); + $qb->select('*') + ->from('preview_locations'); + $result = $qb->executeQuery(); + + $set = []; + + while ($row = $result->fetch()) { + // Iterate over all the rows with duplicated rows + $id = $row['id']; + + if (isset($set[$row['bucket_name'] . '_' . $row['object_store_name']])) { + // duplicate + $authoritativeId = $set[$row['bucket_name'] . '_' . $row['object_store_name']]; + $qb = $this->connection->getQueryBuilder(); + $qb->select('id') + ->from('preview_locations') + ->where($qb->expr()->eq('bucket_name', $qb->createNamedParameter($row['bucket_name']))) + ->andWhere($qb->expr()->eq('object_store_name', $qb->createNamedParameter($row['object_store_name']))) + ->andWhere($qb->expr()->neq('id', $qb->createNamedParameter($authoritativeId))); + + $result = $qb->executeQuery(); + while ($row = $result->fetch()) { + // Update previews entries to the now de-duplicated id + $qb = $this->connection->getQueryBuilder(); + $qb->update('previews') + ->set('location_id', $qb->createNamedParameter($id)) + ->where($qb->expr()->eq('id', $qb->createNamedParameter($row['id']))); + $qb->executeStatement(); + + $qb = $this->connection->getQueryBuilder(); + $qb->delete('preview_locations') + ->where($qb->expr()->eq('id', $qb->createNamedParameter($row['id']))); + $qb->executeStatement(); + } + break; + } + $set[$row['bucket_name'] . '_' . $row['object_store_name']] = $row['id']; + } + } +} diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index d14d80b23f76c..e883f4198330d 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -1530,6 +1530,8 @@ 'OC\\Core\\Migrations\\Version32000Date20250731062008' => $baseDir . '/core/Migrations/Version32000Date20250731062008.php', 'OC\\Core\\Migrations\\Version32000Date20250806110519' => $baseDir . '/core/Migrations/Version32000Date20250806110519.php', 'OC\\Core\\Migrations\\Version33000Date20250819110529' => $baseDir . '/core/Migrations/Version33000Date20250819110529.php', + 'OC\\Core\\Migrations\\Version33000Date20251023110529' => $baseDir . '/core/Migrations/Version33000Date20251023110529.php', + 'OC\\Core\\Migrations\\Version33000Date20251023120529' => $baseDir . '/core/Migrations/Version33000Date20251023120529.php', 'OC\\Core\\Notification\\CoreNotifier' => $baseDir . '/core/Notification/CoreNotifier.php', 'OC\\Core\\ResponseDefinitions' => $baseDir . '/core/ResponseDefinitions.php', 'OC\\Core\\Service\\CronService' => $baseDir . '/core/Service/CronService.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index fd607dfc14a06..a24c51fbf3700 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -1571,6 +1571,8 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Core\\Migrations\\Version32000Date20250731062008' => __DIR__ . '/../../..' . '/core/Migrations/Version32000Date20250731062008.php', 'OC\\Core\\Migrations\\Version32000Date20250806110519' => __DIR__ . '/../../..' . '/core/Migrations/Version32000Date20250806110519.php', 'OC\\Core\\Migrations\\Version33000Date20250819110529' => __DIR__ . '/../../..' . '/core/Migrations/Version33000Date20250819110529.php', + 'OC\\Core\\Migrations\\Version33000Date20251023110529' => __DIR__ . '/../../..' . '/core/Migrations/Version33000Date20251023110529.php', + 'OC\\Core\\Migrations\\Version33000Date20251023120529' => __DIR__ . '/../../..' . '/core/Migrations/Version33000Date20251023120529.php', 'OC\\Core\\Notification\\CoreNotifier' => __DIR__ . '/../../..' . '/core/Notification/CoreNotifier.php', 'OC\\Core\\ResponseDefinitions' => __DIR__ . '/../../..' . '/core/ResponseDefinitions.php', 'OC\\Core\\Service\\CronService' => __DIR__ . '/../../..' . '/core/Service/CronService.php', diff --git a/lib/private/AppFramework/Http/Dispatcher.php b/lib/private/AppFramework/Http/Dispatcher.php index ee20dee6a4541..317c6917e6c10 100644 --- a/lib/private/AppFramework/Http/Dispatcher.php +++ b/lib/private/AppFramework/Http/Dispatcher.php @@ -204,7 +204,7 @@ private function executeController(Controller $controller, string $methodName): try { $response = \call_user_func_array([$controller, $methodName], $arguments); } catch (\TypeError $e) { - // Only intercept TypeErrors occuring on the first line, meaning that the invocation of the controller method failed. + // Only intercept TypeErrors occurring on the first line, meaning that the invocation of the controller method failed. // Any other TypeError happens inside the controller method logic and should be logged as normal. if ($e->getFile() === $this->reflector->getFile() && $e->getLine() === $this->reflector->getStartLine()) { $this->logger->debug('Failed to call controller method: ' . $e->getMessage(), ['exception' => $e]); diff --git a/lib/private/DB/SchemaWrapper.php b/lib/private/DB/SchemaWrapper.php index 0d5b20405130f..19a0bc576f260 100644 --- a/lib/private/DB/SchemaWrapper.php +++ b/lib/private/DB/SchemaWrapper.php @@ -8,8 +8,11 @@ use Doctrine\DBAL\Exception; use Doctrine\DBAL\Platforms\AbstractPlatform; +use Doctrine\DBAL\Platforms\OraclePlatform; use Doctrine\DBAL\Schema\Schema; use OCP\DB\ISchemaWrapper; +use OCP\Server; +use Psr\Log\LoggerInterface; class SchemaWrapper implements ISchemaWrapper { /** @var Connection */ @@ -131,4 +134,18 @@ public function getTables() { public function getDatabasePlatform() { return $this->connection->getDatabasePlatform(); } + + public function dropAutoincrementColumn(string $table, string $column): void { + $tableObj = $this->schema->getTable($this->connection->getPrefix() . $table); + $tableObj->modifyColumn('id', ['autoincrement' => false]); + $platform = $this->getDatabasePlatform(); + if ($platform instanceof OraclePlatform) { + try { + $this->connection->executeStatement('DROP TRIGGER "' . $this->connection->getPrefix() . $table . '_AI_PK"'); + $this->connection->executeStatement('DROP SEQUENCE "' . $this->connection->getPrefix() . $table . '_SEQ"'); + } catch (Exception $e) { + Server::get(LoggerInterface::class)->error($e->getMessage(), ['exception' => $e]); + } + } + } } diff --git a/lib/private/Preview/Db/Preview.php b/lib/private/Preview/Db/Preview.php index 8d1c6d9f72aaa..d76435e23aaca 100644 --- a/lib/private/Preview/Db/Preview.php +++ b/lib/private/Preview/Db/Preview.php @@ -17,14 +17,16 @@ /** * Preview entity mapped to the oc_previews and oc_preview_locations table. * + * @method string getId() + * @method void setId(string $id) * @method int getFileId() Get the file id of the original file. * @method void setFileId(int $fileId) * @method int getStorageId() Get the storage id of the original file. * @method void setStorageId(int $fileId) * @method int getOldFileId() Get the old location in the file-cache table, for legacy compatibility. * @method void setOldFileId(int $oldFileId) - * @method int getLocationId() Get the location id in the preview_locations table. Only set when using an object store as primary storage. - * @method void setLocationId(int $locationId) + * @method string getLocationId() Get the location id in the preview_locations table. Only set when using an object store as primary storage. + * @method void setLocationId(string $locationId) * @method string|null getBucketName() Get the bucket name where the preview is stored. This is stored in the preview_locations table. * @method string|null getObjectStoreName() Get the object store name where the preview is stored. This is stored in the preview_locations table. * @method int getWidth() Get the width of the preview. @@ -46,7 +48,7 @@ * @method string getEtag() Get the etag of the preview. * @method void setEtag(string $etag) * @method string|null getVersion() Get the version for files_versions_s3 - * @method void setVersionId(int $versionId) + * @method void setVersionId(string $versionId) * @method bool|null getIs() Get the version for files_versions_s3 * @method bool isEncrypted() Get whether the preview is encrypted. At the moment every preview is unencrypted. * @method void setEncrypted(bool $encrypted) @@ -57,7 +59,7 @@ class Preview extends Entity { protected ?int $fileId = null; protected ?int $oldFileId = null; protected ?int $storageId = null; - protected ?int $locationId = null; + protected ?string $locationId = null; protected ?string $bucketName = null; protected ?string $objectStoreName = null; protected ?int $width = null; @@ -72,14 +74,15 @@ class Preview extends Entity { protected ?bool $cropped = null; protected ?string $etag = null; protected ?string $version = null; - protected ?int $versionId = null; + protected ?string $versionId = null; protected ?bool $encrypted = null; public function __construct() { + $this->addType('id', Types::STRING); $this->addType('fileId', Types::BIGINT); $this->addType('storageId', Types::BIGINT); $this->addType('oldFileId', Types::BIGINT); - $this->addType('locationId', Types::BIGINT); + $this->addType('locationId', Types::STRING); $this->addType('width', Types::INTEGER); $this->addType('height', Types::INTEGER); $this->addType('mimetypeId', Types::INTEGER); diff --git a/lib/private/Preview/Db/PreviewMapper.php b/lib/private/Preview/Db/PreviewMapper.php index e6ca2e720f358..2a5342d7edbd6 100644 --- a/lib/private/Preview/Db/PreviewMapper.php +++ b/lib/private/Preview/Db/PreviewMapper.php @@ -15,6 +15,7 @@ use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\Files\IMimeTypeLoader; use OCP\IDBConnection; +use OCP\Snowflake\IGenerator; use Override; /** @@ -29,6 +30,7 @@ class PreviewMapper extends QBMapper { public function __construct( IDBConnection $db, private readonly IMimeTypeLoader $mimeTypeLoader, + private readonly IGenerator $snowflake, ) { parent::__construct($db, self::TABLE_NAME, Preview::class); } @@ -50,13 +52,15 @@ public function insert(Entity $entity): Entity { if ($preview->getVersion() !== null && $preview->getVersion() !== '') { $qb = $this->db->getQueryBuilder(); + $id = $this->snowflake->nextId(); $qb->insert(self::VERSION_TABLE_NAME) ->values([ + 'id' => $id, 'version' => $preview->getVersion(), 'file_id' => $preview->getFileId(), ]) ->executeStatement(); - $entity->setVersionId($qb->getLastInsertId()); + $entity->setVersionId($id); } return parent::insert($preview); } @@ -148,7 +152,13 @@ protected function joinLocation(IQueryBuilder $qb): IQueryBuilder { )); } - public function getLocationId(string $bucket, string $objectStore): int { + /** + * Get the location id corresponding to the $bucket and $objectStore. Create one + * if not existing yet. + * + * @throws Exception + */ + public function getLocationId(string $bucket, string $objectStore): string { $qb = $this->db->getQueryBuilder(); $result = $qb->select('id') ->from(self::LOCATION_TABLE_NAME) @@ -157,14 +167,33 @@ public function getLocationId(string $bucket, string $objectStore): int { ->executeQuery(); $data = $result->fetchOne(); if ($data) { - return $data; + return (string)$data; } else { - $qb->insert(self::LOCATION_TABLE_NAME) - ->values([ - 'bucket_name' => $qb->createNamedParameter($bucket), - 'object_store_name' => $qb->createNamedParameter($objectStore), - ])->executeStatement(); - return $qb->getLastInsertId(); + try { + $id = $this->snowflake->nextId(); + $qb->insert(self::LOCATION_TABLE_NAME) + ->values([ + 'id' => $qb->createNamedParameter($id), + 'bucket_name' => $qb->createNamedParameter($bucket), + 'object_store_name' => $qb->createNamedParameter($objectStore), + ])->executeStatement(); + return $id; + } catch (Exception $e) { + if ($e->getReason() === Exception::REASON_UNIQUE_CONSTRAINT_VIOLATION) { + // Fetch again as there seems to be another entry added meanwhile + $result = $qb->select('id') + ->from(self::LOCATION_TABLE_NAME) + ->where($qb->expr()->eq('bucket_name', $qb->createNamedParameter($bucket))) + ->andWhere($qb->expr()->eq('object_store_name', $qb->createNamedParameter($objectStore))) + ->executeQuery(); + $data = $result->fetchOne(); + if ($data) { + return (string)$data; + } + } + + throw $e; + } } } diff --git a/lib/private/Preview/Generator.php b/lib/private/Preview/Generator.php index 8707c257e7b8a..8980e0b2d002a 100644 --- a/lib/private/Preview/Generator.php +++ b/lib/private/Preview/Generator.php @@ -23,6 +23,7 @@ use OCP\IStreamImage; use OCP\Preview\BeforePreviewFetchedEvent; use OCP\Preview\IVersionedPreviewFile; +use OCP\Snowflake\IGenerator; use Psr\Log\LoggerInterface; class Generator { @@ -37,6 +38,7 @@ public function __construct( private LoggerInterface $logger, private PreviewMapper $previewMapper, private StorageFactory $storageFactory, + private IGenerator $snowflakeGenerator, ) { } @@ -348,6 +350,7 @@ private function generateProviderPreview(File $file, int $width, int $height, bo try { $previewEntry = new Preview(); + $previewEntry->setId($this->snowflakeGenerator->nextId()); $previewEntry->setFileId($file->getId()); $previewEntry->setStorageId($file->getMountPoint()->getNumericStorageId()); $previewEntry->setSourceMimeType($file->getMimeType()); @@ -360,7 +363,6 @@ private function generateProviderPreview(File $file, int $width, int $height, bo $previewEntry->setMimetype($preview->dataMimeType()); $previewEntry->setEtag($file->getEtag()); $previewEntry->setMtime((new \DateTime())->getTimestamp()); - $previewEntry->setSize(0); return $this->savePreview($previewEntry, $preview); } catch (NotPermittedException) { throw new NotFoundException(); @@ -502,6 +504,7 @@ private function generatePreview( } $previewEntry = new Preview(); + $previewEntry->setId($this->snowflakeGenerator->nextId()); $previewEntry->setFileId($file->getId()); $previewEntry->setStorageId($file->getMountPoint()->getNumericStorageId()); $previewEntry->setWidth($width); @@ -514,7 +517,6 @@ private function generatePreview( $previewEntry->setMimeType($preview->dataMimeType()); $previewEntry->setEtag($file->getEtag()); $previewEntry->setMtime((new \DateTime())->getTimestamp()); - $previewEntry->setSize(0); if ($cacheResult) { $previewEntry = $this->savePreview($previewEntry, $preview); return new PreviewFile($previewEntry, $this->storageFactory, $this->previewMapper); @@ -530,26 +532,20 @@ private function generatePreview( * @throws \OCP\DB\Exception */ public function savePreview(Preview $previewEntry, IImage $preview): Preview { - $previewEntry = $this->previewMapper->insert($previewEntry); - // we need to save to DB first - try { - if ($preview instanceof IStreamImage) { - $size = $this->storageFactory->writePreview($previewEntry, $preview->resource()); - } else { - $stream = fopen('php://temp', 'w+'); - fwrite($stream, $preview->data()); - rewind($stream); - $size = $this->storageFactory->writePreview($previewEntry, $stream); - } - if (!$size) { - throw new \RuntimeException('Unable to write preview file'); - } - } catch (\Exception $e) { - $this->previewMapper->delete($previewEntry); - throw $e; + if ($preview instanceof IStreamImage) { + $size = $this->storageFactory->writePreview($previewEntry, $preview->resource()); + } else { + $stream = fopen('php://temp', 'w+'); + fwrite($stream, $preview->data()); + rewind($stream); + $size = $this->storageFactory->writePreview($previewEntry, $stream); + } + if (!$size) { + throw new \RuntimeException('Unable to write preview file'); } $previewEntry->setSize($size); - return $this->previewMapper->update($previewEntry); + + return $this->previewMapper->insert($previewEntry); } } diff --git a/lib/private/Preview/Storage/LocalPreviewStorage.php b/lib/private/Preview/Storage/LocalPreviewStorage.php index bd5e1a97818cd..3cf626280b2e6 100644 --- a/lib/private/Preview/Storage/LocalPreviewStorage.php +++ b/lib/private/Preview/Storage/LocalPreviewStorage.php @@ -22,6 +22,7 @@ use OCP\IAppConfig; use OCP\IConfig; use OCP\IDBConnection; +use OCP\Snowflake\IGenerator; use Override; use Psr\Log\LoggerInterface; use RecursiveDirectoryIterator; @@ -38,6 +39,7 @@ public function __construct( private readonly IDBConnection $connection, private readonly IMimeTypeDetector $mimeTypeDetector, private readonly LoggerInterface $logger, + private readonly IGenerator $generator, ) { $this->instanceId = $this->config->getSystemValueString('instanceid'); $this->rootFolder = $this->config->getSystemValue('datadirectory', OC::$SERVERROOT . '/data'); @@ -118,6 +120,7 @@ public function scan(): int { $this->logger->error('Unable to parse preview information for ' . $file->getRealPath()); continue; } + $preview->setId($this->generator->nextId()); try { $preview->setSize($file->getSize()); $preview->setMtime($file->getMtime()); diff --git a/lib/private/PreviewManager.php b/lib/private/PreviewManager.php index c230322b2e0f9..95a4f01b7a8e0 100644 --- a/lib/private/PreviewManager.php +++ b/lib/private/PreviewManager.php @@ -23,6 +23,7 @@ use OCP\IConfig; use OCP\IPreview; use OCP\Preview\IProviderV2; +use OCP\Snowflake\IGenerator; use Psr\Container\ContainerInterface; use Psr\Container\NotFoundExceptionInterface; use Psr\Log\LoggerInterface; @@ -141,6 +142,7 @@ private function getGenerator(): Generator { $this->container->get(LoggerInterface::class), $this->container->get(PreviewMapper::class), $this->container->get(StorageFactory::class), + $this->container->get(IGenerator::class), ); } return $this->generator; diff --git a/lib/public/AppFramework/Db/Entity.php b/lib/public/AppFramework/Db/Entity.php index 3094070af5f15..75dcd3e7766c2 100644 --- a/lib/public/AppFramework/Db/Entity.php +++ b/lib/public/AppFramework/Db/Entity.php @@ -19,10 +19,8 @@ * @psalm-consistent-constructor */ abstract class Entity { - /** - * @var int - */ - public $id; + /** @var int $id */ + public $id = null; private array $_updatedFields = []; /** @var array */ diff --git a/lib/public/DB/ISchemaWrapper.php b/lib/public/DB/ISchemaWrapper.php index dcf22b52d3d9e..8995fd5bfb1ee 100644 --- a/lib/public/DB/ISchemaWrapper.php +++ b/lib/public/DB/ISchemaWrapper.php @@ -90,4 +90,11 @@ public function getTableNamesWithoutPrefix(); * @since 23.0.0 */ public function getDatabasePlatform(); + + /** + * Drop autoincrement from an existing table of the database. + * + * @since 33.0.0 + */ + public function dropAutoincrementColumn(string $table, string $column): void; } diff --git a/tests/lib/Preview/GeneratorTest.php b/tests/lib/Preview/GeneratorTest.php index ceaf483a6588c..074a3272dbf32 100644 --- a/tests/lib/Preview/GeneratorTest.php +++ b/tests/lib/Preview/GeneratorTest.php @@ -22,6 +22,7 @@ use OCP\Preview\BeforePreviewFetchedEvent; use OCP\Preview\IProviderV2; use OCP\Preview\IVersionedPreviewFile; +use OCP\Snowflake\IGenerator; use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\Attributes\TestWith; use PHPUnit\Framework\MockObject\MockObject; @@ -41,6 +42,7 @@ class GeneratorTest extends TestCase { private LoggerInterface&MockObject $logger; private StorageFactory&MockObject $storageFactory; private PreviewMapper&MockObject $previewMapper; + private IGenerator&MockObject $snowflakeGenerator; protected function setUp(): void { parent::setUp(); @@ -52,6 +54,7 @@ protected function setUp(): void { $this->logger = $this->createMock(LoggerInterface::class); $this->previewMapper = $this->createMock(PreviewMapper::class); $this->storageFactory = $this->createMock(StorageFactory::class); + $this->snowflakeGenerator = $this->createMock(IGenerator::class); $this->generator = new Generator( $this->config, @@ -61,6 +64,7 @@ protected function setUp(): void { $this->logger, $this->previewMapper, $this->storageFactory, + $this->snowflakeGenerator, ); } diff --git a/tests/lib/Preview/MovePreviewJobTest.php b/tests/lib/Preview/MovePreviewJobTest.php index 69f730474e7bd..ee1bb0f9ab4af 100644 --- a/tests/lib/Preview/MovePreviewJobTest.php +++ b/tests/lib/Preview/MovePreviewJobTest.php @@ -24,6 +24,7 @@ use OCP\IConfig; use OCP\IDBConnection; use OCP\Server; +use OCP\Snowflake\IGenerator; use PHPUnit\Framework\Attributes\TestDox; use PHPUnit\Framework\MockObject\MockObject; use Psr\Log\LoggerInterface; @@ -123,6 +124,7 @@ public function testMigrationLegacyPath(): void { $this->mimeTypeDetector, $this->mimeTypeLoader, $this->logger, + Server::get(IGenerator::class), Server::get(IAppDataFactory::class), ); $this->invokePrivate($job, 'run', [[]]); @@ -155,6 +157,7 @@ public function testMigrationPath(): void { $this->mimeTypeDetector, $this->mimeTypeLoader, $this->logger, + Server::get(IGenerator::class), Server::get(IAppDataFactory::class) ); $this->invokePrivate($job, 'run', [[]]); @@ -195,6 +198,7 @@ public function testMigrationPathWithVersion(): void { $this->mimeTypeDetector, $this->mimeTypeLoader, $this->logger, + Server::get(IGenerator::class), Server::get(IAppDataFactory::class) ); $this->invokePrivate($job, 'run', [[]]); diff --git a/tests/lib/Preview/PreviewMapperTest.php b/tests/lib/Preview/PreviewMapperTest.php index 0fc5420e23503..5ed1c6f8451ad 100644 --- a/tests/lib/Preview/PreviewMapperTest.php +++ b/tests/lib/Preview/PreviewMapperTest.php @@ -14,16 +14,28 @@ use OC\Preview\Db\PreviewMapper; use OCP\IDBConnection; use OCP\Server; +use OCP\Snowflake\IGenerator; use Test\TestCase; #[\PHPUnit\Framework\Attributes\Group('DB')] class PreviewMapperTest extends TestCase { private PreviewMapper $previewMapper; private IDBConnection $connection; + private IGenerator $snowflake; public function setUp(): void { $this->previewMapper = Server::get(PreviewMapper::class); $this->connection = Server::get(IDBConnection::class); + $this->snowflake = Server::get(IGenerator::class); + + $qb = $this->connection->getQueryBuilder(); + $qb->delete('preview_locations')->executeStatement(); + + $qb = $this->connection->getQueryBuilder(); + $qb->delete('preview_versions')->executeStatement(); + + $qb = $this->connection->getQueryBuilder(); + $qb->delete('previews')->executeStatement(); } public function testGetAvailablePreviews(): void { @@ -51,15 +63,17 @@ private function createPreviewForFileId(int $fileId, ?int $bucket = null): void $locationId = null; if ($bucket) { $qb = $this->connection->getQueryBuilder(); + $locationId = $this->snowflake->nextId(); $qb->insert('preview_locations') ->values([ + 'id' => $locationId, 'bucket_name' => $qb->createNamedParameter('preview-' . $bucket), 'object_store_name' => $qb->createNamedParameter('default'), ]); $qb->executeStatement(); - $locationId = $qb->getLastInsertId(); } $preview = new Preview(); + $preview->setId($this->snowflake->nextId()); $preview->setFileId($fileId); $preview->setStorageId(1); $preview->setCropped(true); diff --git a/tests/lib/Preview/PreviewServiceTest.php b/tests/lib/Preview/PreviewServiceTest.php index 09f6a891402d9..a09b07380aabc 100644 --- a/tests/lib/Preview/PreviewServiceTest.php +++ b/tests/lib/Preview/PreviewServiceTest.php @@ -14,7 +14,7 @@ use OC\Preview\Db\PreviewMapper; use OC\Preview\PreviewService; use OCP\Server; -use PHPUnit\Framework\Attributes\CoversClass; +use OCP\Snowflake\IGenerator; use PHPUnit\Framework\TestCase; #[CoversClass(PreviewService::class)] @@ -22,10 +22,12 @@ class PreviewServiceTest extends TestCase { private PreviewService $previewService; private PreviewMapper $previewMapper; + private IGenerator $snowflakeGenerator; protected function setUp(): void { $this->previewService = Server::get(PreviewService::class); $this->previewMapper = Server::get(PreviewMapper::class); + $this->snowflakeGenerator = Server::get(IGenerator::class); $this->previewService->deleteAll(); } @@ -36,6 +38,7 @@ public function tearDown(): void { public function testGetAvailableFileIds(): void { foreach (range(1, 20) as $i) { $preview = new Preview(); + $preview->setId($this->snowflakeGenerator->nextId()); $preview->setFileId($i % 10); $preview->setStorageId(1); $preview->setWidth($i); diff --git a/version.php b/version.php index 7d52f0b5b1e5c..5633d19142f39 100644 --- a/version.php +++ b/version.php @@ -9,7 +9,7 @@ // between betas, final and RCs. This is _not_ the public version number. Reset minor/patch level // when updating major/minor version number. -$OC_Version = [33, 0, 0, 2]; +$OC_Version = [33, 0, 0, 3]; // The human-readable string $OC_VersionString = '33.0.0 dev';