Skip to content

feat(updater): download resume w/ transfer statistics #571

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
come-nc merged 6 commits into from
Feb 10, 2025
Merged

feat(updater): download resume w/ transfer statistics #571

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
f22b063
feat(updater): resume aborted downloads and log stats
joshtrichards Jun 23, 2024
ee1b78a
fix: adjust download resume variable/property handling
joshtrichards Jul 4, 2024
a4970e2
fix: cast to int
joshtrichards Jul 4, 2024
a13f913
fix: reset download progress just in case
joshtrichards Jul 16, 2024
c9cfaec
fix: cast to int
joshtrichards Jan 19, 2025
f9e62dc
chore: Reapply coding standards
come-nc Feb 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 72 additions & 13 deletions index.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ class Updater {
private bool $updateAvailable = false;
private ?string $requestID = null;
private bool $disabled = false;
private int $previousProgress = 0;

/**
* Updater constructor
Expand Down Expand Up @@ -544,30 +545,56 @@ private function getUpdateServerResponse(): array {
/**
* Downloads the nextcloud folder to $DATADIR/updater-$instanceid/downloads/$filename
*
* Logs download progress
* Resumes incomplete downloads if possible
* Supports outbound proxy usage
* Logs download statistics upon completion
*
* TODO: Provide download progress in real-time (in both CLI and Web modes)
*
* @throws \Exception
*/
public function downloadUpdate(): void {
$this->silentLog('[info] downloadUpdate()');

$response = $this->getUpdateServerResponse();

$storageLocation = $this->getUpdateDirectoryLocation() . '/updater-' . $this->getConfigOptionMandatoryString('instanceid') . '/downloads/';
if (file_exists($storageLocation)) {
$this->silentLog('[info] storage location exists');
$this->recursiveDelete($storageLocation);
}
$state = mkdir($storageLocation, 0750, true);
if ($state === false) {
throw new \Exception('Could not mkdir storage location');
}

if (!isset($response['url']) || !is_string($response['url'])) {
throw new \Exception('Response from update server is missing url');
}

$fp = fopen($storageLocation . basename($response['url']), 'w+');
$storageLocation = $this->getUpdateDirectoryLocation() . '/updater-' . $this->getConfigOptionMandatoryString('instanceid') . '/downloads/';
$saveLocation = $storageLocation . basename($response['url']);
$this->previousProgress = 0;

$ch = curl_init($response['url']);

if (!file_exists($storageLocation)) {
$state = mkdir($storageLocation, 0750, true);
if ($state === false) {
throw new \Exception('Could not mkdir storage location');
}
$this->silentLog('[info] storage location created');
} else {
$this->silentLog('[info] storage location already exists');
// clean-up leftover extracted content from any prior runs, but leave any downloaded Archives alone
if (file_exists($storageLocation . 'nextcloud/')) {
$this->silentLog('[info] extracted Archive location exists');
$this->recursiveDelete($storageLocation . 'nextcloud/');
}
Comment on lines +580 to +583
Copy link

@artonge artonge Jul 23, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could we instead delete any file that is not $saveLocation to clean out old downloads?

// see if there's an existing incomplete download to resume
if (is_file($saveLocation)) {
$size = filesize($saveLocation);
$range = $size . '-';
curl_setopt($ch, CURLOPT_RANGE, $range);
$this->silentLog('[info] previous download found; resuming from ' . $this->formatBytes($size));
}
}

$fp = fopen($saveLocation, 'a');
curl_setopt_array($ch, [
CURLOPT_RETURNTRANSFER => true,
CURLOPT_NOPROGRESS => false,
CURLOPT_PROGRESSFUNCTION => [$this, 'downloadProgressCallback'],
CURLOPT_FILE => $fp,
CURLOPT_USERAGENT => 'Nextcloud Updater',
]);
Expand All @@ -584,7 +611,7 @@ public function downloadUpdate(): void {
throw new \Exception('Curl error: ' . curl_error($ch));
}
$httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
if ($httpCode !== 200) {
if ($httpCode !== 200 && $httpCode !== 206) {
$statusCodes = [
400 => 'Bad request',
401 => 'Unauthorized',
Expand All @@ -611,13 +638,45 @@ public function downloadUpdate(): void {
$message .= ' - URL: ' . htmlentities($response['url']);

throw new \Exception($message);
} else {
// download succeeded
$info = curl_getinfo($ch);
$this->silentLog('[info] download stats: size=' . $this->formatBytes((int)$info['size_download']) . ' bytes; total_time=' . round($info['total_time'], 2) . ' secs; avg speed=' . $this->formatBytes((int)$info['speed_download']) . '/sec');
}

curl_close($ch);
fclose($fp);

$this->silentLog('[info] end of downloadUpdate()');
}

private function downloadProgressCallback(\CurlHandle $resource, int $download_size, int $downloaded, int $upload_size, int $uploaded): void {
if ($download_size !== 0) {
$progress = (int)round($downloaded * 100 / $download_size);
if ($progress > $this->previousProgress) {
$this->previousProgress = $progress;
// log every 2% increment for the first 10% then only log every 10% increment after that
if ($progress % 10 === 0 || ($progress < 10 && $progress % 2 === 0)) {
$this->silentLog("[info] download progress: $progress% (" . $this->formatBytes($downloaded) . ' of ' . $this->formatBytes($download_size) . ')');
}
}
}
}

private function formatBytes(int $bytes, int $precision = 2): string {
$units = ['B', 'KB', 'MB', 'GB', 'TB'];

$bytes = max($bytes, 0);
$pow = floor(($bytes ? log($bytes) : 0) / log(1024));
$pow = min($pow, count($units) - 1);

// Uncomment one of the following alternatives
$bytes /= pow(1024, $pow);
// $bytes /= (1 << (10 * $pow));

return round($bytes, $precision) . $units[(int)$pow];
}

/**
* @throws \Exception
*/
Expand Down
85 changes: 72 additions & 13 deletions lib/Updater.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ class Updater {
private bool $updateAvailable = false;
private ?string $requestID = null;
private bool $disabled = false;
private int $previousProgress = 0;

/**
* Updater constructor
Expand Down Expand Up @@ -526,30 +527,56 @@ private function getUpdateServerResponse(): array {
/**
* Downloads the nextcloud folder to $DATADIR/updater-$instanceid/downloads/$filename
*
* Logs download progress
* Resumes incomplete downloads if possible
* Supports outbound proxy usage
* Logs download statistics upon completion
*
* TODO: Provide download progress in real-time (in both CLI and Web modes)
*
* @throws \Exception
*/
public function downloadUpdate(): void {
$this->silentLog('[info] downloadUpdate()');

$response = $this->getUpdateServerResponse();

$storageLocation = $this->getUpdateDirectoryLocation() . '/updater-' . $this->getConfigOptionMandatoryString('instanceid') . '/downloads/';
if (file_exists($storageLocation)) {
$this->silentLog('[info] storage location exists');
$this->recursiveDelete($storageLocation);
}
$state = mkdir($storageLocation, 0750, true);
if ($state === false) {
throw new \Exception('Could not mkdir storage location');
}

if (!isset($response['url']) || !is_string($response['url'])) {
throw new \Exception('Response from update server is missing url');
}

$fp = fopen($storageLocation . basename($response['url']), 'w+');
$storageLocation = $this->getUpdateDirectoryLocation() . '/updater-' . $this->getConfigOptionMandatoryString('instanceid') . '/downloads/';
$saveLocation = $storageLocation . basename($response['url']);
$this->previousProgress = 0;

$ch = curl_init($response['url']);

if (!file_exists($storageLocation)) {
$state = mkdir($storageLocation, 0750, true);
if ($state === false) {
throw new \Exception('Could not mkdir storage location');
}
$this->silentLog('[info] storage location created');
} else {
$this->silentLog('[info] storage location already exists');
// clean-up leftover extracted content from any prior runs, but leave any downloaded Archives alone
if (file_exists($storageLocation . 'nextcloud/')) {
$this->silentLog('[info] extracted Archive location exists');
$this->recursiveDelete($storageLocation . 'nextcloud/');
}
// see if there's an existing incomplete download to resume
if (is_file($saveLocation)) {
$size = filesize($saveLocation);
$range = $size . '-';
curl_setopt($ch, CURLOPT_RANGE, $range);
$this->silentLog('[info] previous download found; resuming from ' . $this->formatBytes($size));
}
}

$fp = fopen($saveLocation, 'a');
curl_setopt_array($ch, [
CURLOPT_RETURNTRANSFER => true,
CURLOPT_NOPROGRESS => false,
CURLOPT_PROGRESSFUNCTION => [$this, 'downloadProgressCallback'],
CURLOPT_FILE => $fp,
CURLOPT_USERAGENT => 'Nextcloud Updater',
]);
Expand All @@ -566,7 +593,7 @@ public function downloadUpdate(): void {
throw new \Exception('Curl error: ' . curl_error($ch));
}
$httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
if ($httpCode !== 200) {
if ($httpCode !== 200 && $httpCode !== 206) {
$statusCodes = [
400 => 'Bad request',
401 => 'Unauthorized',
Expand All @@ -593,13 +620,45 @@ public function downloadUpdate(): void {
$message .= ' - URL: ' . htmlentities($response['url']);

throw new \Exception($message);
} else {
// download succeeded
$info = curl_getinfo($ch);
$this->silentLog('[info] download stats: size=' . $this->formatBytes((int)$info['size_download']) . ' bytes; total_time=' . round($info['total_time'], 2) . ' secs; avg speed=' . $this->formatBytes((int)$info['speed_download']) . '/sec');
}

curl_close($ch);
fclose($fp);

$this->silentLog('[info] end of downloadUpdate()');
}

private function downloadProgressCallback(\CurlHandle $resource, int $download_size, int $downloaded, int $upload_size, int $uploaded): void {
if ($download_size !== 0) {
$progress = (int)round($downloaded * 100 / $download_size);
if ($progress > $this->previousProgress) {
$this->previousProgress = $progress;
// log every 2% increment for the first 10% then only log every 10% increment after that
if ($progress % 10 === 0 || ($progress < 10 && $progress % 2 === 0)) {
$this->silentLog("[info] download progress: $progress% (" . $this->formatBytes($downloaded) . ' of ' . $this->formatBytes($download_size) . ')');
}
}
}
}

private function formatBytes(int $bytes, int $precision = 2): string {
$units = ['B', 'KB', 'MB', 'GB', 'TB'];

$bytes = max($bytes, 0);
$pow = floor(($bytes ? log($bytes) : 0) / log(1024));
$pow = min($pow, count($units) - 1);

// Uncomment one of the following alternatives
$bytes /= pow(1024, $pow);
// $bytes /= (1 << (10 * $pow));

return round($bytes, $precision) . $units[(int)$pow];
}

/**
* @throws \Exception
*/
Expand Down
Binary file modified updater.phar
View file
Open in desktop
Binary file not shown.