diff --git a/index.php b/index.php index dde1c943..dd787d2f 100644 --- a/index.php +++ b/index.php @@ -47,6 +47,8 @@ public function accept(): bool { } +use CurlHandle; + class Updater { private string $baseDir; private array $configValues = []; @@ -55,6 +57,7 @@ class Updater { private bool $updateAvailable = false; private ?string $requestID = null; private bool $disabled = false; + private int $previousProgress = 0; /** * Updater constructor @@ -278,6 +281,7 @@ private function getExpectedElementsList(): array { 'COPYING-AGPL', 'occ', 'db_structure.xml', + 'REUSE.toml', ]; return array_merge($expected, $this->getAppDirectories()); } @@ -514,20 +518,7 @@ private function getUpdateServerResponse(): array { $this->silentLog('[info] updateURL: ' . $updateURL); // Download update response - $curl = curl_init(); - curl_setopt_array($curl, [ - CURLOPT_RETURNTRANSFER => 1, - CURLOPT_URL => $updateURL, - CURLOPT_USERAGENT => 'Nextcloud Updater', - ]); - - if ($this->getConfigOption('proxy') !== null) { - curl_setopt_array($curl, [ - CURLOPT_PROXY => $this->getConfigOptionString('proxy'), - CURLOPT_PROXYUSERPWD => $this->getConfigOptionString('proxyuserpwd'), - CURLOPT_HTTPPROXYTUNNEL => $this->getConfigOption('proxy') ? 1 : 0, - ]); - } + $curl = $this->getCurl($updateURL); /** @var false|string $response */ $response = curl_exec($curl); @@ -558,26 +549,79 @@ private function getUpdateServerResponse(): array { public function downloadUpdate(): void { $this->silentLog('[info] downloadUpdate()'); - $response = $this->getUpdateServerResponse(); + $downloadURLs = $this->getDownloadURLs(); + $this->silentLog('[info] will try to download archive from: ' . implode(', ', $downloadURLs)); - $storageLocation = $this->getUpdateDirectoryLocation() . '/updater-'.$this->getConfigOptionMandatoryString('instanceid') . '/downloads/'; - if (file_exists($storageLocation)) { - $this->silentLog('[info] storage location exists'); - $this->recursiveDelete($storageLocation); + $storageLocation = $this->getUpdateDirectoryLocation() . '/updater-' . $this->getConfigOptionMandatoryString('instanceid') . '/downloads/'; + + if (!file_exists($storageLocation)) { + $state = mkdir($storageLocation, 0750, true); + if ($state === false) { + throw new \Exception('Could not mkdir storage location'); + } + $this->silentLog('[info] storage location created'); + } else { + $this->silentLog('[info] storage location already exists'); + // clean-up leftover extracted content from any prior runs, but leave any downloaded Archives alone + if (file_exists($storageLocation . 'nextcloud/')) { + $this->silentLog('[info] extracted Archive location exists'); + $this->recursiveDelete($storageLocation . 'nextcloud/'); + } } - $state = mkdir($storageLocation, 0750, true); - if ($state === false) { - throw new \Exception('Could not mkdir storage location'); + + foreach ($downloadURLs as $url) { + $this->previousProgress = 0; + $saveLocation = $storageLocation . basename($url); + if ($this->downloadArchive($url, $saveLocation)) { + return; + } + } + + throw new \Exception('All downloads failed. See updater logs for more information.'); + } + + private function getDownloadURLs(): array { + $response = $this->getUpdateServerResponse(); + $downloadURLs = []; + if (!isset($response['downloads']) || !is_array($response['downloads'])) { + if (isset($response['url']) && is_string($response['url'])) { + // Compatibility with previous verison of updater_server + $ext = pathinfo($response['url'], PATHINFO_EXTENSION); + $response['downloads'] = [ + $ext => [$response['url']] + ]; + } else { + throw new \Exception('Response from update server is missing download URLs'); + } + } + foreach ($response['downloads'] as $format => $urls) { + if (!$this->isAbleToDecompress($format)) { + continue; + } + foreach ($urls as $url) { + if (!is_string($url)) { + continue; + } + $downloadURLs[] = $url; + } } - if (!isset($response['url']) || !is_string($response['url'])) { - throw new \Exception('Response from update server is missing url'); + if (empty($downloadURLs)) { + throw new \Exception('Your PHP install is not able to decompress any archive. Try to install modules like zip or bzip.'); + } + + return array_unique($downloadURLs); + + } + + private function getCurl(string $url): CurlHandle { + $ch = curl_init($url); + if ($ch === false) { + throw new \Exception('Fail to open cUrl handler'); } - $fp = fopen($storageLocation . basename($response['url']), 'w+'); - $ch = curl_init($response['url']); curl_setopt_array($ch, [ - CURLOPT_FILE => $fp, + CURLOPT_RETURNTRANSFER => true, CURLOPT_USERAGENT => 'Nextcloud Updater', CURLOPT_FOLLOWLOCATION => 1, CURLOPT_MAXREDIRS => 2, @@ -591,42 +635,88 @@ public function downloadUpdate(): void { ]); } + return $ch; + } + + private function downloadArchive(string $fromUrl, string $toLocation): bool { + $ch = $this->getCurl($fromUrl); + + // see if there's an existing incomplete download to resume + if (is_file($toLocation)) { + $size = (int)filesize($toLocation); + $range = $size . '-'; + curl_setopt($ch, CURLOPT_RANGE, $range); + $this->silentLog('[info] previous download found; resuming from ' . $this->formatBytes($size)); + } + + $fp = fopen($toLocation, 'ab'); + if ($fp === false) { + throw new \Exception('Fail to open file in ' . $toLocation); + } + + curl_setopt_array($ch, [ + CURLOPT_NOPROGRESS => false, + CURLOPT_PROGRESSFUNCTION => [$this, 'downloadProgressCallback'], + CURLOPT_FILE => $fp, + ]); + if (curl_exec($ch) === false) { throw new \Exception('Curl error: ' . curl_error($ch)); } - $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE); - if ($httpCode !== 200) { - $statusCodes = [ - 400 => 'Bad request', - 401 => 'Unauthorized', - 403 => 'Forbidden', - 404 => 'Not Found', - 500 => 'Internal Server Error', - 502 => 'Bad Gateway', - 503 => 'Service Unavailable', - 504 => 'Gateway Timeout', - ]; - - $message = 'Download failed'; - if (is_int($httpCode) && isset($statusCodes[$httpCode])) { - $message .= ' - ' . $statusCodes[$httpCode] . ' (HTTP ' . $httpCode . ')'; - } else { - $message .= ' - HTTP status code: ' . (string)$httpCode; - } - - $curlErrorMessage = curl_error($ch); - if (!empty($curlErrorMessage)) { - $message .= ' - curl error message: ' . $curlErrorMessage; - } - $message .= ' - URL: ' . htmlentities($response['url']); + $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE); + if ($httpCode !== 200 && $httpCode !== 206) { + fclose($fp); + unlink($toLocation); + $this->silentLog('[warn] fail to download archive from ' . $fromUrl . '. Error: ' . $httpCode . ' ' . curl_error($ch)); + curl_close($ch); - throw new \Exception($message); + return false; } + // download succeeded + $info = curl_getinfo($ch); + $this->silentLog('[info] download stats: size=' . $this->formatBytes((int)$info['size_download']) . ' bytes; total_time=' . round($info['total_time'], 2) . ' secs; avg speed=' . $this->formatBytes((int)$info['speed_download']) . '/sec'); + curl_close($ch); fclose($fp); $this->silentLog('[info] end of downloadUpdate()'); + return true; + } + + /** + * Check if PHP is able to decompress archive format + */ + private function isAbleToDecompress(string $ext): bool { + // Only zip is supported for now + return $ext === 'zip' && extension_loaded($ext); + } + + private function downloadProgressCallback(CurlHandle $resource, int $download_size, int $downloaded, int $upload_size, int $uploaded): void { + if ($download_size !== 0) { + $progress = (int)round($downloaded * 100 / $download_size); + if ($progress > $this->previousProgress) { + $this->previousProgress = $progress; + // log every 2% increment for the first 10% then only log every 10% increment after that + if ($progress % 10 === 0 || ($progress < 10 && $progress % 2 === 0)) { + $this->silentLog("[info] download progress: $progress% (" . $this->formatBytes($downloaded) . ' of ' . $this->formatBytes($download_size) . ')'); + } + } + } + } + + private function formatBytes(int $bytes, int $precision = 2): string { + $units = ['B', 'KiB', 'MiB', 'GiB', 'TiB']; + + $bytes = max($bytes, 0); + $pow = floor(($bytes ? log($bytes) : 0) / log(1024)); + $pow = min($pow, count($units) - 1); + + // Uncomment one of the following alternatives + $bytes /= pow(1024, $pow); + // $bytes /= (1 << (10 * $pow)); + + return round($bytes, $precision) . $units[(int)$pow]; } /** @@ -638,13 +728,14 @@ private function getDownloadedFilePath(): string { $filesInStorageLocation = scandir($storageLocation); $files = array_values(array_filter($filesInStorageLocation, function (string $path) { - return $path !== '.' && $path !== '..'; + // Match files with - in the name and extension (*-*.*) + return preg_match('/^.*-.*\..*$/i', $path); })); // only the downloaded archive if (count($files) !== 1) { throw new \Exception('There are more files than the downloaded archive in the downloads/ folder.'); } - return $storageLocation . '/' . $files[0]; + return $storageLocation . $files[0]; } /** diff --git a/lib/Updater.php b/lib/Updater.php index 1320e874..70f06bcc 100644 --- a/lib/Updater.php +++ b/lib/Updater.php @@ -9,6 +9,8 @@ namespace NC\Updater; +use CurlHandle; + class Updater { private string $baseDir; private array $configValues = []; @@ -17,6 +19,7 @@ class Updater { private bool $updateAvailable = false; private ?string $requestID = null; private bool $disabled = false; + private int $previousProgress = 0; /** * Updater constructor @@ -240,6 +243,7 @@ private function getExpectedElementsList(): array { 'COPYING-AGPL', 'occ', 'db_structure.xml', + 'REUSE.toml', ]; return array_merge($expected, $this->getAppDirectories()); } @@ -476,20 +480,7 @@ private function getUpdateServerResponse(): array { $this->silentLog('[info] updateURL: ' . $updateURL); // Download update response - $curl = curl_init(); - curl_setopt_array($curl, [ - CURLOPT_RETURNTRANSFER => 1, - CURLOPT_URL => $updateURL, - CURLOPT_USERAGENT => 'Nextcloud Updater', - ]); - - if ($this->getConfigOption('proxy') !== null) { - curl_setopt_array($curl, [ - CURLOPT_PROXY => $this->getConfigOptionString('proxy'), - CURLOPT_PROXYUSERPWD => $this->getConfigOptionString('proxyuserpwd'), - CURLOPT_HTTPPROXYTUNNEL => $this->getConfigOption('proxy') ? 1 : 0, - ]); - } + $curl = $this->getCurl($updateURL); /** @var false|string $response */ $response = curl_exec($curl); @@ -520,26 +511,79 @@ private function getUpdateServerResponse(): array { public function downloadUpdate(): void { $this->silentLog('[info] downloadUpdate()'); - $response = $this->getUpdateServerResponse(); + $downloadURLs = $this->getDownloadURLs(); + $this->silentLog('[info] will try to download archive from: ' . implode(', ', $downloadURLs)); - $storageLocation = $this->getUpdateDirectoryLocation() . '/updater-'.$this->getConfigOptionMandatoryString('instanceid') . '/downloads/'; - if (file_exists($storageLocation)) { - $this->silentLog('[info] storage location exists'); - $this->recursiveDelete($storageLocation); + $storageLocation = $this->getUpdateDirectoryLocation() . '/updater-' . $this->getConfigOptionMandatoryString('instanceid') . '/downloads/'; + + if (!file_exists($storageLocation)) { + $state = mkdir($storageLocation, 0750, true); + if ($state === false) { + throw new \Exception('Could not mkdir storage location'); + } + $this->silentLog('[info] storage location created'); + } else { + $this->silentLog('[info] storage location already exists'); + // clean-up leftover extracted content from any prior runs, but leave any downloaded Archives alone + if (file_exists($storageLocation . 'nextcloud/')) { + $this->silentLog('[info] extracted Archive location exists'); + $this->recursiveDelete($storageLocation . 'nextcloud/'); + } } - $state = mkdir($storageLocation, 0750, true); - if ($state === false) { - throw new \Exception('Could not mkdir storage location'); + + foreach ($downloadURLs as $url) { + $this->previousProgress = 0; + $saveLocation = $storageLocation . basename($url); + if ($this->downloadArchive($url, $saveLocation)) { + return; + } + } + + throw new \Exception('All downloads failed. See updater logs for more information.'); + } + + private function getDownloadURLs(): array { + $response = $this->getUpdateServerResponse(); + $downloadURLs = []; + if (!isset($response['downloads']) || !is_array($response['downloads'])) { + if (isset($response['url']) && is_string($response['url'])) { + // Compatibility with previous verison of updater_server + $ext = pathinfo($response['url'], PATHINFO_EXTENSION); + $response['downloads'] = [ + $ext => [$response['url']] + ]; + } else { + throw new \Exception('Response from update server is missing download URLs'); + } } + foreach ($response['downloads'] as $format => $urls) { + if (!$this->isAbleToDecompress($format)) { + continue; + } + foreach ($urls as $url) { + if (!is_string($url)) { + continue; + } + $downloadURLs[] = $url; + } + } + + if (empty($downloadURLs)) { + throw new \Exception('Your PHP install is not able to decompress any archive. Try to install modules like zip or bzip.'); + } + + return array_unique($downloadURLs); - if (!isset($response['url']) || !is_string($response['url'])) { - throw new \Exception('Response from update server is missing url'); + } + + private function getCurl(string $url): CurlHandle { + $ch = curl_init($url); + if ($ch === false) { + throw new \Exception('Fail to open cUrl handler'); } - $fp = fopen($storageLocation . basename($response['url']), 'w+'); - $ch = curl_init($response['url']); curl_setopt_array($ch, [ - CURLOPT_FILE => $fp, + CURLOPT_RETURNTRANSFER => true, CURLOPT_USERAGENT => 'Nextcloud Updater', CURLOPT_FOLLOWLOCATION => 1, CURLOPT_MAXREDIRS => 2, @@ -553,42 +597,88 @@ public function downloadUpdate(): void { ]); } + return $ch; + } + + private function downloadArchive(string $fromUrl, string $toLocation): bool { + $ch = $this->getCurl($fromUrl); + + // see if there's an existing incomplete download to resume + if (is_file($toLocation)) { + $size = (int)filesize($toLocation); + $range = $size . '-'; + curl_setopt($ch, CURLOPT_RANGE, $range); + $this->silentLog('[info] previous download found; resuming from ' . $this->formatBytes($size)); + } + + $fp = fopen($toLocation, 'ab'); + if ($fp === false) { + throw new \Exception('Fail to open file in ' . $toLocation); + } + + curl_setopt_array($ch, [ + CURLOPT_NOPROGRESS => false, + CURLOPT_PROGRESSFUNCTION => [$this, 'downloadProgressCallback'], + CURLOPT_FILE => $fp, + ]); + if (curl_exec($ch) === false) { throw new \Exception('Curl error: ' . curl_error($ch)); } - $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE); - if ($httpCode !== 200) { - $statusCodes = [ - 400 => 'Bad request', - 401 => 'Unauthorized', - 403 => 'Forbidden', - 404 => 'Not Found', - 500 => 'Internal Server Error', - 502 => 'Bad Gateway', - 503 => 'Service Unavailable', - 504 => 'Gateway Timeout', - ]; - - $message = 'Download failed'; - if (is_int($httpCode) && isset($statusCodes[$httpCode])) { - $message .= ' - ' . $statusCodes[$httpCode] . ' (HTTP ' . $httpCode . ')'; - } else { - $message .= ' - HTTP status code: ' . (string)$httpCode; - } - $curlErrorMessage = curl_error($ch); - if (!empty($curlErrorMessage)) { - $message .= ' - curl error message: ' . $curlErrorMessage; - } - - $message .= ' - URL: ' . htmlentities($response['url']); + $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE); + if ($httpCode !== 200 && $httpCode !== 206) { + fclose($fp); + unlink($toLocation); + $this->silentLog('[warn] fail to download archive from ' . $fromUrl . '. Error: ' . $httpCode . ' ' . curl_error($ch)); + curl_close($ch); - throw new \Exception($message); + return false; } + // download succeeded + $info = curl_getinfo($ch); + $this->silentLog('[info] download stats: size=' . $this->formatBytes((int)$info['size_download']) . ' bytes; total_time=' . round($info['total_time'], 2) . ' secs; avg speed=' . $this->formatBytes((int)$info['speed_download']) . '/sec'); + curl_close($ch); fclose($fp); $this->silentLog('[info] end of downloadUpdate()'); + return true; + } + + /** + * Check if PHP is able to decompress archive format + */ + private function isAbleToDecompress(string $ext): bool { + // Only zip is supported for now + return $ext === 'zip' && extension_loaded($ext); + } + + private function downloadProgressCallback(CurlHandle $resource, int $download_size, int $downloaded, int $upload_size, int $uploaded): void { + if ($download_size !== 0) { + $progress = (int)round($downloaded * 100 / $download_size); + if ($progress > $this->previousProgress) { + $this->previousProgress = $progress; + // log every 2% increment for the first 10% then only log every 10% increment after that + if ($progress % 10 === 0 || ($progress < 10 && $progress % 2 === 0)) { + $this->silentLog("[info] download progress: $progress% (" . $this->formatBytes($downloaded) . ' of ' . $this->formatBytes($download_size) . ')'); + } + } + } + } + + private function formatBytes(int $bytes, int $precision = 2): string { + $units = ['B', 'KiB', 'MiB', 'GiB', 'TiB']; + + $bytes = max($bytes, 0); + $pow = floor(($bytes ? log($bytes) : 0) / log(1024)); + $pow = min($pow, count($units) - 1); + + // Uncomment one of the following alternatives + $bytes /= pow(1024, $pow); + // $bytes /= (1 << (10 * $pow)); + + return round($bytes, $precision) . $units[(int)$pow]; } /** @@ -600,13 +690,14 @@ private function getDownloadedFilePath(): string { $filesInStorageLocation = scandir($storageLocation); $files = array_values(array_filter($filesInStorageLocation, function (string $path) { - return $path !== '.' && $path !== '..'; + // Match files with - in the name and extension (*-*.*) + return preg_match('/^.*-.*\..*$/i', $path); })); // only the downloaded archive if (count($files) !== 1) { throw new \Exception('There are more files than the downloaded archive in the downloads/ folder.'); } - return $storageLocation . '/' . $files[0]; + return $storageLocation . $files[0]; } /** diff --git a/tests/features/cli.feature b/tests/features/cli.feature index a4fd2807..a0eceabd 100644 --- a/tests/features/cli.feature +++ b/tests/features/cli.feature @@ -28,7 +28,7 @@ Feature: CLI updater And there is an update to version 25.0.503 available When the CLI updater is run Then the return code should not be 0 - And the output should contain "Download failed - Not Found (HTTP 404)" + And the output should contain "All downloads failed." And the installed version should be 25.0.0 And maintenance mode should be off And upgrade is not required diff --git a/updater.phar b/updater.phar index 357087da..ab9b1183 100755 Binary files a/updater.phar and b/updater.phar differ diff --git a/vendor/composer/installed.php b/vendor/composer/installed.php index f151d63c..480c81a8 100644 --- a/vendor/composer/installed.php +++ b/vendor/composer/installed.php @@ -3,7 +3,7 @@ 'name' => '__root__', 'pretty_version' => 'dev-master', 'version' => 'dev-master', - 'reference' => 'b5f1ef738c155e5410703c6c75fd1e8ff3faacfd', + 'reference' => 'bbd6e8e521f3fd1d02d34e3c9eda83c4c8d4c460', 'type' => 'library', 'install_path' => __DIR__ . '/../../', 'aliases' => array(), @@ -13,7 +13,7 @@ '__root__' => array( 'pretty_version' => 'dev-master', 'version' => 'dev-master', - 'reference' => 'b5f1ef738c155e5410703c6c75fd1e8ff3faacfd', + 'reference' => 'bbd6e8e521f3fd1d02d34e3c9eda83c4c8d4c460', 'type' => 'library', 'install_path' => __DIR__ . '/../../', 'aliases' => array(), diff --git a/vendor/composer/platform_check.php b/vendor/composer/platform_check.php index 580fa960..d2225c7d 100644 --- a/vendor/composer/platform_check.php +++ b/vendor/composer/platform_check.php @@ -19,8 +19,7 @@ echo 'Composer detected issues in your platform:' . PHP_EOL.PHP_EOL . str_replace('You are running '.PHP_VERSION.'.', '', implode(PHP_EOL, $issues)) . PHP_EOL.PHP_EOL; } } - trigger_error( - 'Composer detected issues in your platform: ' . implode(' ', $issues), - E_USER_ERROR + throw new \RuntimeException( + 'Composer detected issues in your platform: ' . implode(' ', $issues) ); }