Skip to content

feat(previews): allow ffmpeg to connect direct for AWS S3 buckets #53634

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
invario wants to merge 1 commit into
base: master
Choose a base branch
from
+121 −58
Open

feat(previews): allow ffmpeg to connect direct for AWS S3 buckets #53634

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 31 additions & 0 deletions apps/files_external/lib/Lib/Storage/AmazonS3.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -757,4 +757,35 @@ public function writeStream(string $path, $stream, ?int $size = null): int {

return $size;
}

/**
* Generates and returns a presigned URL that expires after set duration
*
*/
public function getDirectDownload(string $path): array|false {
$command = $this->getConnection()->getCommand('GetObject', [
'Bucket' => $this->bucket,
'Key' => $path,
]);
$duration = '+10 minutes';
$expiration = new \DateTime();
$expiration->modify($duration);

// generate a presigned URL that expires after $duration time
$request = $this->getConnection()->createPresignedRequest($command, $duration, []);
try {
$presignedUrl = (string)$request->getUri();
} catch (S3Exception $exception) {
$this->logger->error($exception->getMessage(), [
'app' => 'files_external',
'exception' => $exception,
]);
}
$result = [
'url' => $presignedUrl,
'presigned' => true,
'expiration' => $expiration,
];
return $result;
}
}
148 changes: 90 additions & 58 deletions lib/private/Preview/Movie.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,26 @@ public function isAvailable(FileInfo $file): bool {
return is_string($this->binary);
}

private function connectDirect(File $file): string|false {
if (stream_get_meta_data($file->fopen('r'))['seekable'] !== true) {
return false;
}

// Checks for availability to access the video file directly via HTTP/HTTPS.
// Returns a string containing URL if available. Only implemented and tested
// with Amazon S3 currently. In all other cases, return false. ffmpeg
// supports other protocols so this function may expand in the future.
$gddValues = $file->getStorage()->getDirectDownload($file->getName());

if (is_array($gddValues)) {
if (array_key_exists('url', $gddValues) && array_key_exists('presigned', $gddValues)) {
$directUrl = (str_starts_with($gddValues['url'], 'http') && ($gddValues['presigned'] === true)) ? $gddValues['url'] : false;
return $directUrl;
}
}
return false;
}

/**
* {@inheritDoc}
*/
Expand All @@ -54,74 +74,87 @@ public function getThumbnail(File $file, int $maxX, int $maxY): ?IImage {

$result = null;

$connectDirect = $this->connectDirect($file);

// Timestamps to make attempts to generate a still
$timeAttempts = [5, 1, 0];

// By default, download $sizeAttempts from the file along with
// the 'moov' atom.
// Example bitrates in the higher range:
// 4K HDR H265 60 FPS = 75 Mbps = 9 MB per second needed for a still
// 1080p H265 30 FPS = 10 Mbps = 1.25 MB per second needed for a still
// 1080p H264 30 FPS = 16 Mbps = 2 MB per second needed for a still
$sizeAttempts = [1024 * 1024 * 10];

if ($this->useTempFile($file)) {
if ($file->getStorage()->isLocal()) {
// Temp file required but file is local, so retrieve $sizeAttempt bytes first,
// and if it doesn't work, retrieve the entire file.
$sizeAttempts[] = null;
// If HTTP/HTTPS direct connect is not available or if the file is encrypted,
// process normally
if (($connectDirect === false) || $file->isEncrypted()) {
// By default, download $sizeAttempts from the file along with
// the 'moov' atom.
// Example bitrates in the higher range:
// 4K HDR H265 60 FPS = 75 Mbps = 9 MB per second needed for a still
// 1080p H265 30 FPS = 10 Mbps = 1.25 MB per second needed for a still
// 1080p H264 30 FPS = 16 Mbps = 2 MB per second needed for a still
$sizeAttempts = [1024 * 1024 * 10];

if ($this->useTempFile($file)) {
if ($file->getStorage()->isLocal()) {
// Temp file required but file is local, so retrieve $sizeAttempt bytes first,
// and if it doesn't work, retrieve the entire file.
$sizeAttempts[] = null;
}
} else {
// Temp file is not required and file is local so retrieve entire file.
$sizeAttempts = [null];
}
} else {
// Temp file is not required and file is local so retrieve entire file.
$sizeAttempts = [null];
}

foreach ($sizeAttempts as $size) {
$absPath = false;
// File is remote, generate a sparse file
if (!$file->getStorage()->isLocal()) {
$absPath = $this->getSparseFile($file, $size);
}
// Defaults to existing routine if generating sparse file fails
if ($absPath === false) {
$absPath = $this->getLocalFile($file, $size);
}
if ($absPath === false) {
Server::get(LoggerInterface::class)->error(
'Failed to get local file to generate thumbnail for: ' . $file->getPath(),
['app' => 'core']
);
return null;
}
foreach ($sizeAttempts as $size) {
$absPath = false;
// File is remote, generate a sparse file
if (!$file->getStorage()->isLocal()) {
$absPath = $this->getSparseFile($file, $size);
}
// Defaults to existing routine if generating sparse file fails
if ($absPath === false) {
$absPath = $this->getLocalFile($file, $size);
}
if ($absPath === false) {
Server::get(LoggerInterface::class)->error(
'Failed to get local file to generate thumbnail for: ' . $file->getPath(),
['app' => 'core']
);
return null;
}

// Attempt still image grabs from selected timestamps
foreach ($timeAttempts as $timeStamp) {
$result = $this->generateThumbNail($maxX, $maxY, $absPath, $timeStamp);
if ($result !== null) {
break;
}
Server::get(LoggerInterface::class)->debug(
'Movie preview generation attempt failed'
. ', file=' . $file->getPath()
. ', time=' . $timeStamp
. ', size=' . ($size ?? 'entire file'),
['app' => 'core']
);
}

$this->cleanTmpFiles();

// Attempt still image grabs from selected timestamps
foreach ($timeAttempts as $timeStamp) {
$result = $this->generateThumbNail($maxX, $maxY, $absPath, $timeStamp);
if ($result !== null) {
Server::get(LoggerInterface::class)->debug(
'Movie preview generation attempt success'
. ', file=' . $file->getPath()
. ', time=' . $timeStamp
. ', size=' . ($size ?? 'entire file'),
['app' => 'core']
);
break;
}
Server::get(LoggerInterface::class)->debug(
'Movie preview generation attempt failed'
. ', file=' . $file->getPath()
. ', time=' . $timeStamp
. ', size=' . ($size ?? 'entire file'),
['app' => 'core']
);
}

$this->cleanTmpFiles();

if ($result !== null) {
Server::get(LoggerInterface::class)->debug(
'Movie preview generation attempt success'
. ', file=' . $file->getPath()
. ', time=' . $timeStamp
. ', size=' . ($size ?? 'entire file'),
['app' => 'core']
);
break;
} else {
// HTTP/HTTPS direct connect is available so pass the URL directly to ffmpeg
foreach ($timeAttempts as $timeStamp) {
$result = $this->generateThumbNail($maxX, $maxY, $connectDirect, $timeStamp);
if ($result !== null) {
break;
}
}

}
if ($result === null) {
Server::get(LoggerInterface::class)->error(
Expand Down Expand Up @@ -330,7 +363,6 @@ private function generateThumbNail(int $maxX, int $maxY, string $absPath, int $s
}
}


unlink($tmpPath);
return null;
}
Expand Down
Loading