diff --git a/lib/Command/CirclesMaintenance.php b/lib/Command/CirclesMaintenance.php index d788f19b3..47eefcc25 100644 --- a/lib/Command/CirclesMaintenance.php +++ b/lib/Command/CirclesMaintenance.php @@ -34,6 +34,7 @@ use OC\Core\Command\Base; use OCA\Circles\Db\CoreRequestBuilder; +use OCA\Circles\Exceptions\MaintenanceException; use OCA\Circles\Service\MaintenanceService; use OCA\Circles\Service\OutputService; use Symfony\Component\Console\Input\InputInterface; @@ -84,7 +85,7 @@ protected function configure() { parent::configure(); $this->setName('circles:maintenance') ->setDescription('Clean stuff, keeps the app running') - ->addOption('level', '', InputOption::VALUE_REQUIRED, 'level of maintenance', '0') + ->addOption('level', '', InputOption::VALUE_REQUIRED, 'level of maintenance', '3') ->addOption( 'reset', '', InputOption::VALUE_NONE, 'reset Circles; remove all data related to the App' ) @@ -157,7 +158,12 @@ protected function execute(InputInterface $input, OutputInterface $output): int } $this->outputService->setOccOutput($output); - $this->maintenanceService->runMaintenance($level); + for ($i = 1; $i <= $level; $i++) { + try { + $this->maintenanceService->runMaintenance($i); + } catch (MaintenanceException $e) { + } + } $output->writeln(''); $output->writeln('done'); diff --git a/lib/Cron/Maintenance.php b/lib/Cron/Maintenance.php index 45399261c..3744f93cd 100644 --- a/lib/Cron/Maintenance.php +++ b/lib/Cron/Maintenance.php @@ -30,7 +30,10 @@ namespace OCA\Circles\Cron; +use ArtificialOwl\MySmallPhpTools\Model\SimpleDataStore; use OC\BackgroundJob\TimedJob; +use OCA\Circles\Exceptions\MaintenanceException; +use OCA\Circles\Service\ConfigService; use OCA\Circles\Service\MaintenanceService; @@ -45,14 +48,30 @@ class Maintenance extends TimedJob { /** @var MaintenanceService */ private $maintenanceService; + /** @var ConfigService */ + private $configService; + + + static $DELAY = + [ + 1 => 60, // every minute + 2 => 300, // every 5 minutes + 3 => 3600, // every hour + 4 => 75400, // every day + 5 => 432000 // evey week + ]; /** * Cache constructor. */ - public function __construct(MaintenanceService $maintenanceService) { + public function __construct( + MaintenanceService $maintenanceService, + ConfigService $configService + ) { $this->setInterval(10); $this->maintenanceService = $maintenanceService; + $this->configService = $configService; } @@ -60,7 +79,67 @@ public function __construct(MaintenanceService $maintenanceService) { * @param $argument */ protected function run($argument) { - $this->maintenanceService->runMaintenance(3); + $this->runMaintenances(); + } + + + /** + * + */ + private function runMaintenances(): void { + $last = new SimpleDataStore(); + $last->json($this->configService->getAppValue(ConfigService::MAINTENANCE_UPDATE)); + + $last->sInt('maximum', $this->maximumLevelBasedOnTime(($last->gInt('5') === 0))); + for ($i = 5; $i > 0; $i--) { + if ($this->canRunLevel($i, $last)) { + try { + $this->maintenanceService->runMaintenance($i); + } catch (MaintenanceException $e) { + continue; + } + $last->sInt((string)$i, time()); + } + } + + $this->configService->setAppValue(ConfigService::MAINTENANCE_UPDATE, json_encode($last)); + } + + + /** + * @param bool $force + * + * @return int + */ + private function maximumLevelBasedOnTime(bool $force = false): int { + $currentHour = (int)date('H'); + $currentDay = (int)date('N'); + $isWeekEnd = ($currentDay >= 6); + + if ($currentHour > 2 && $currentHour < 5 && ($isWeekEnd || $force)) { + return 5; + } + + if ($currentHour > 1 && $currentHour < 6) { + return 4; + } + + return 3; + } + + + private function canRunLevel(int $level, SimpleDataStore $last): bool { + if ($last->gInt('maximum') < $level) { + return false; + } + + $now = time(); + $timeLastRun = $last->gInt((string)$level); + if ($timeLastRun === 0) { + return true; + } + + return ($timeLastRun + self::$DELAY[$level] < $now); } } diff --git a/lib/Db/CoreRequestBuilder.php b/lib/Db/CoreRequestBuilder.php index 16bc00187..a0f90b629 100644 --- a/lib/Db/CoreRequestBuilder.php +++ b/lib/Db/CoreRequestBuilder.php @@ -129,6 +129,7 @@ class CoreRequestBuilder { 'instance', 'interface', 'severity', + 'retry', 'status', 'creation' ], diff --git a/lib/Db/EventWrapperRequest.php b/lib/Db/EventWrapperRequest.php index cb00dd589..ef9cc3f7b 100644 --- a/lib/Db/EventWrapperRequest.php +++ b/lib/Db/EventWrapperRequest.php @@ -58,6 +58,7 @@ public function save(EventWrapper $wrapper): void { ->setValue('instance', $qb->createNamedParameter($wrapper->getInstance())) ->setValue('interface', $qb->createNamedParameter($wrapper->getInterface())) ->setValue('severity', $qb->createNamedParameter($wrapper->getSeverity())) + ->setValue('retry', $qb->createNamedParameter($wrapper->getRetry())) ->setValue('status', $qb->createNamedParameter($wrapper->getStatus())) ->setValue('creation', $qb->createNamedParameter($wrapper->getCreation())); @@ -70,7 +71,8 @@ public function save(EventWrapper $wrapper): void { public function update(EventWrapper $wrapper): void { $qb = $this->getEventWrapperUpdateSql(); $qb->set('result', $qb->createNamedParameter(json_encode($wrapper->getResult()))) - ->set('status', $qb->createNamedParameter($wrapper->getStatus())); + ->set('status', $qb->createNamedParameter($wrapper->getStatus())) + ->set('retry', $qb->createNamedParameter($wrapper->getRetry())); $qb->limitToInstance($wrapper->getInstance()); $qb->limitToToken($wrapper->getToken()); @@ -98,9 +100,11 @@ public function updateAll(string $token, int $status): void { * * @return EventWrapper[] */ - public function getFailedEvents(): array { + public function getFailedEvents(array $retryRange): array { $qb = $this->getEventWrapperSelectSql(); $qb->limitInt('status', EventWrapper::STATUS_FAILED); + $qb->gt('retry', $retryRange[0], true); + $qb->lt('retry', $retryRange[1]); return $this->getItemsFromRequest($qb); } diff --git a/lib/Exceptions/MaintenanceException.php b/lib/Exceptions/MaintenanceException.php new file mode 100644 index 000000000..65a0f9478 --- /dev/null +++ b/lib/Exceptions/MaintenanceException.php @@ -0,0 +1,46 @@ + + * @copyright 2021 + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OCA\Circles\Exceptions; + + +use Exception; + + +/** + * Class MaintenanceException + * + * @package OCA\Circles\Exceptions + */ +class MaintenanceException extends Exception { + +} + + diff --git a/lib/Migration/Version0022Date20220526113601.php b/lib/Migration/Version0022Date20220526113601.php index 944d86498..228c05f2d 100644 --- a/lib/Migration/Version0022Date20220526113601.php +++ b/lib/Migration/Version0022Date20220526113601.php @@ -383,6 +383,12 @@ public function changeSchema(IOutput $output, Closure $schemaClosure, array $opt 'notnull' => false ] ); + $table->addColumn( + 'retry', 'integer', [ + 'length' => 3, + 'notnull' => false + ] + ); $table->addColumn( 'status', 'integer', [ 'length' => 3, diff --git a/lib/Migration/Version0022Date20220623224231.php b/lib/Migration/Version0022Date20220626112233.php similarity index 95% rename from lib/Migration/Version0022Date20220623224231.php rename to lib/Migration/Version0022Date20220626112233.php index a6f424103..e19163ca2 100644 --- a/lib/Migration/Version0022Date20220623224231.php +++ b/lib/Migration/Version0022Date20220626112233.php @@ -44,7 +44,7 @@ * * @package OCA\Circles\Migration */ -class Version0022Date20220623224231 extends SimpleMigrationStep { +class Version0022Date20220626112233 extends SimpleMigrationStep { /** @@ -75,6 +75,14 @@ public function changeSchema(IOutput $output, Closure $schemaClosure, array $opt ] ); } + if (!$table->hasColumn('retry')) { + $table->addColumn( + 'retry', 'integer', [ + 'length' => 3, + 'notnull' => false + ] + ); + } } if ($schema->hasTable('circles_member')) { diff --git a/lib/Model/Federated/EventWrapper.php b/lib/Model/Federated/EventWrapper.php index 957e29a4d..c6f73fb15 100644 --- a/lib/Model/Federated/EventWrapper.php +++ b/lib/Model/Federated/EventWrapper.php @@ -37,7 +37,6 @@ use ArtificialOwl\MySmallPhpTools\Model\SimpleDataStore; use ArtificialOwl\MySmallPhpTools\Traits\TArrayTools; use JsonSerializable; -use OCA\Circles\Exceptions\UnknownInterfaceException; /** @@ -75,6 +74,9 @@ class EventWrapper implements INC22QueryRow, JsonSerializable { /** @var int */ private $severity = FederatedEvent::SEVERITY_LOW; + /** @var int */ + private $retry = 0; + /** @var int */ private $status = 0; @@ -207,6 +209,24 @@ public function setSeverity(int $severity): self { return $this; } + /** + * @param int $retry + * + * @return EventWrapper + */ + public function setRetry(int $retry): self { + $this->retry = $retry; + + return $this; + } + + /** + * @return int + */ + public function getRetry(): int { + return $this->retry; + } + /** * @return int diff --git a/lib/Service/ConfigService.php b/lib/Service/ConfigService.php index 2e8be7fed..3d3b5533b 100644 --- a/lib/Service/ConfigService.php +++ b/lib/Service/ConfigService.php @@ -93,6 +93,8 @@ class ConfigService { const MIGRATION_BYPASS = 'migration_bypass'; const MIGRATION_22 = 'migration_22'; const MIGRATION_RUN = 'migration_run'; + const MAINTENANCE_UPDATE = 'maintenance_update'; + const MAINTENANCE_RUN = 'maintenance_run'; const LOOPBACK_TMP_ID = 'loopback_tmp_id'; const LOOPBACK_TMP_SCHEME = 'loopback_tmp_scheme'; @@ -151,6 +153,8 @@ class ConfigService { self::MIGRATION_BYPASS => '0', self::MIGRATION_22 => '0', self::MIGRATION_RUN => '0', + self::MAINTENANCE_UPDATE => '[]', + self::MAINTENANCE_RUN => '0', self::FORCE_NC_BASE => '', self::TEST_NC_BASE => '', diff --git a/lib/Service/EventWrapperService.php b/lib/Service/EventWrapperService.php index 7d0e14e5d..287f097c9 100644 --- a/lib/Service/EventWrapperService.php +++ b/lib/Service/EventWrapperService.php @@ -52,6 +52,17 @@ class EventWrapperService extends NC22Signature { use TStringTools; + const RETRY_ASAP = 'asap'; + const RETRY_HOURLY = 'hourly'; + const RETRY_DAILY = 'daily'; + const RETRY_ERROR = 100; + static $RETRIES = [ + 'asap' => [0, 5], + 'hourly' => [5, 150], + 'daily' => [150, 300] + ]; + + /** @var EventWrapperRequest */ private $eventWrapperRequest; @@ -122,7 +133,7 @@ public function manageWrapper(EventWrapper $wrapper): int { } $status = EventWrapper::STATUS_FAILED; - + $retry = $wrapper->getRetry(); try { if ($this->configService->isLocalInstance($wrapper->getInstance())) { $gs = $this->federatedEventService->getFederatedItem($wrapper->getEvent(), false); @@ -132,6 +143,7 @@ public function manageWrapper(EventWrapper $wrapper): int { } $status = EventWrapper::STATUS_DONE; } catch (Exception $e) { + $retry++; } if ($wrapper->getSeverity() !== FederatedEvent::SEVERITY_HIGH) { @@ -139,6 +151,7 @@ public function manageWrapper(EventWrapper $wrapper): int { } $wrapper->setStatus($status); + $wrapper->setRetry($retry); $wrapper->setResult($wrapper->getEvent()->getResult()); $this->eventWrapperRequest->update($wrapper); @@ -148,10 +161,10 @@ public function manageWrapper(EventWrapper $wrapper): int { /** - * retry failed High Severity FederatedEvent + * @param string $retry */ - public function retry() { - $tokens = $this->getFailedEvents(); + public function retry(string $retry) { + $tokens = $this->getFailedEvents(self::$RETRIES[$retry]); foreach ($tokens as $token) { $this->confirmStatus($token, true); } @@ -159,13 +172,15 @@ public function retry() { /** - * returns token from failed FederatedEvent + * @param array $retryRange + * + * @return array */ - private function getFailedEvents(): array { + private function getFailedEvents(array $retryRange): array { $token = array_map( function(EventWrapper $event): string { return $event->getToken(); - }, $this->eventWrapperRequest->getFailedEvents() + }, $this->eventWrapperRequest->getFailedEvents($retryRange) ); return array_values(array_unique($token)); diff --git a/lib/Service/MaintenanceService.php b/lib/Service/MaintenanceService.php index 460f0f0f0..9a16cac03 100644 --- a/lib/Service/MaintenanceService.php +++ b/lib/Service/MaintenanceService.php @@ -37,6 +37,7 @@ use OCA\Circles\Db\CircleRequest; use OCA\Circles\Db\MemberRequest; use OCA\Circles\Exceptions\InitiatorNotFoundException; +use OCA\Circles\Exceptions\MaintenanceException; use OCA\Circles\Exceptions\RequestBuilderException; use OCA\Circles\Model\Circle; use OCA\Circles\Model\Member; @@ -52,6 +53,9 @@ class MaintenanceService { + const TIMEOUT = 18000; + + /** @var IUserManager */ private $userManager; @@ -61,6 +65,9 @@ class MaintenanceService { /** @var MemberRequest */ private $memberRequest; + /** @var SyncService */ + private $syncService; + /** @var FederatedUserService */ private $federatedUserService; @@ -70,6 +77,9 @@ class MaintenanceService { /** @var CircleService */ private $circleService; + /** @var ConfigService */ + private $configService; + /** @var OutputInterface */ private $output; @@ -84,21 +94,26 @@ class MaintenanceService { * @param FederatedUserService $federatedUserService * @param EventWrapperService $eventWrapperService * @param CircleService $circleService + * @param ConfigService $configService */ public function __construct( IUserManager $userManager, CircleRequest $circleRequest, MemberRequest $memberRequest, + SyncService $syncService, FederatedUserService $federatedUserService, EventWrapperService $eventWrapperService, - CircleService $circleService + CircleService $circleService, + ConfigService $configService ) { $this->userManager = $userManager; $this->circleRequest = $circleRequest; $this->memberRequest = $memberRequest; + $this->syncService = $syncService; $this->federatedUserService = $federatedUserService; $this->eventWrapperService = $eventWrapperService; $this->circleService = $circleService; + $this->configService = $configService; } @@ -111,67 +126,134 @@ public function setOccOutput(OutputInterface $output): void { /** + * level=1 -> run every minute + * level=2 -> run every 5 minutes + * level=3 -> run every hour + * level=4 -> run every day + * level=5 -> run every week * + * @param int $level + * + * @throws MaintenanceException */ - public function runMaintenance(int $level = 0): void { + public function runMaintenance(int $level): void { $this->federatedUserService->bypassCurrentUserCondition(true); + $this->lockMaintenanceRun(); + echo 'running maintenance(' . $level . ')' . "\n"; + + switch ($level) { + case 1: + $this->runMaintenance1(); + break; + case 2: + $this->runMaintenance2(); + break; + case 3: + $this->runMaintenance3(); + break; + case 4: + $this->runMaintenance4(); + break; + case 5: + $this->runMaintenance5(); + break; + } + + $this->configService->setAppValue(ConfigService::MAINTENANCE_RUN, '0'); + } + + + /** + * @throws MaintenanceException + */ + private function lockMaintenanceRun(): void { + $run = $this->configService->getAppValueInt(ConfigService::MAINTENANCE_RUN); + if ($run > time() - self::TIMEOUT) { + throw new MaintenanceException('maintenance already running'); + } + + $this->configService->setAppValue(ConfigService::MAINTENANCE_RUN, (string)time()); + } + + + /** + * every minute + */ + private function runMaintenance1(): void { try { $this->output('remove circles with no owner'); $this->removeCirclesWithNoOwner(); } catch (Exception $e) { } + } + + /** + * every 10 minutes + */ + private function runMaintenance2(): void { try { $this->output('remove members with no circles'); $this->removeMembersWithNoCircles(); } catch (Exception $e) { } - try { - // TODO: waiting for confirmation of a good migration before cleaning orphan shares -// $this->output('remove deprecated shares'); -// $this->removeDeprecatedShares(); + $this->output('retry failed FederatedEvents (asap)'); + $this->eventWrapperService->retry(EventWrapperService::RETRY_ASAP); } catch (Exception $e) { } + } + + /** + * every hour + */ + private function runMaintenance3(): void { try { - $this->output('retry failed FederatedEvents'); - $this->eventWrapperService->retry(); + $this->output('retry failed FederatedEvents (hourly)'); + $this->eventWrapperService->retry(EventWrapperService::RETRY_HOURLY); } catch (Exception $e) { } + } - if ($level < 1) { - return; - } - if ($level < 2) { - return; + /** + * every day + */ + private function runMaintenance4(): void { + try { + $this->output('retry failed FederatedEvents (daily)'); + $this->eventWrapperService->retry(EventWrapperService::RETRY_DAILY); + } catch (Exception $e) { + } + try { + // TODO: waiting for confirmation of a good migration before cleaning orphan shares +// $this->output('remove deprecated shares'); +// $this->removeDeprecatedShares(); + } catch (Exception $e) { } - if ($level < 3) { - return; + try { + $this->output('synchronizing local entities'); + $this->syncService->sync(); + } catch (Exception $e) { } + } -// if ($level < 5) { + /** + * every week + */ + private function runMaintenance5(): void { +// try { // $this->output('refresh displayNames older than 7d'); // // $this->refreshOldDisplayNames(); +// $this->output('refresh DisplayNames'); +// $this->refreshDisplayName(); +// } catch (Exception $e) { // } -// - if ($level < 4) { - return; - } - - if ($level < 5) { - return; - } - try { - $this->output('refresh DisplayNames'); - $this->refreshDisplayName(); - } catch (Exception $e) { - } } diff --git a/lib/Service/MigrationService.php b/lib/Service/MigrationService.php index 03f9cf092..fad569554 100644 --- a/lib/Service/MigrationService.php +++ b/lib/Service/MigrationService.php @@ -91,9 +91,6 @@ class MigrationService { /** @var MemberRequest */ private $memberRequest; - /** @var SyncService */ - private $syncService; - /** @var MembershipService */ private $membershipService; @@ -127,6 +124,7 @@ class MigrationService { * @param IURLGenerator $urlGenerator * @param CircleRequest $circleRequest * @param MemberRequest $memberRequest + * @param MembershipService $membershipService * @param FederatedUserService $federatedUserService * @param CircleService $circleService * @param ContactService $contactService @@ -139,7 +137,6 @@ public function __construct( IURLGenerator $urlGenerator, CircleRequest $circleRequest, MemberRequest $memberRequest, - SyncService $syncService, MembershipService $membershipService, FederatedUserService $federatedUserService, CircleService $circleService, @@ -152,7 +149,6 @@ public function __construct( $this->urlGenerator = $urlGenerator; $this->circleRequest = $circleRequest; $this->memberRequest = $memberRequest; - $this->syncService = $syncService; $this->membershipService = $membershipService; $this->federatedUserService = $federatedUserService; $this->circleService = $circleService; @@ -210,7 +206,6 @@ private function migrationTo22(): void { $this->outputService->output('Migrating to 22'); - $this->syncService->sync(); $this->migrationTo22_Circles(); $this->migrationTo22_Members(); $this->membershipService->resetMemberships('', true); @@ -317,7 +312,6 @@ private function saveGeneratedCircle(Circle $circle): void { $this->circleRequest->save($circle); } catch (InvalidIdException $e) { } - usleep(50); } catch (RequestBuilderException $e) { } }