Skip to content

[stable27] implement optimized getDirectoryContent for DAV #38971

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
blizzz merged 2 commits into from
Jul 11, 2023
Merged

[stable27] implement optimized getDirectoryContent for DAV #38971

Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
98ec2f0
implement optimized getDirectoryContent for DAV
icewind1991 Jun 22, 2023
61cd250
Include PR #38965
solracsf Jun 23, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
implement optimized getDirectoryContent for DAV 
Signed-off-by: Robin Appelman <[email protected]>
  • Loading branch information
@icewind1991 @blizzz
icewind1991 authored and blizzz committed Jul 10, 2023
commit 98ec2f023ffa2fe7d2813768d3e4aa9bb4f10f39
221 changes: 125 additions & 96 deletions lib/private/Files/Storage/DAV.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
use OCP\Diagnostics\IEventLogger;
use OCP\Files\FileInfo;
use OCP\Files\ForbiddenException;
use OCP\Files\IMimeTypeDetector;
use OCP\Files\StorageInvalidException;
use OCP\Files\StorageNotAvailableException;
use OCP\Http\Client\IClientService;
Expand Down Expand Up @@ -93,10 +94,22 @@ class DAV extends Common {
protected $certManager;
protected LoggerInterface $logger;
protected IEventLogger $eventLogger;
protected IMimeTypeDetector $mimeTypeDetector;

/** @var int */
private $timeout;

protected const PROPFIND_PROPS = [
'{DAV:}getlastmodified',
'{DAV:}getcontentlength',
'{DAV:}getcontenttype',
'{http://owncloud.org/ns}permissions',
'{http://open-collaboration-services.org/ns}share-permissions',
'{DAV:}resourcetype',
'{DAV:}getetag',
'{DAV:}quota-available-bytes',
];

/**
* @param array $params
* @throws \Exception
Expand Down Expand Up @@ -141,6 +154,7 @@ public function __construct($params) {
$this->eventLogger = \OC::$server->get(IEventLogger::class);
// This timeout value will be used for the download and upload of files
$this->timeout = \OC::$server->get(IConfig::class)->getSystemValueInt('davstorage.request_timeout', 30);
$this->mimeTypeDetector = \OC::$server->getMimeTypeDetector();
}

protected function init() {
Expand Down Expand Up @@ -239,31 +253,12 @@ public function opendir($path) {
$this->init();
$path = $this->cleanPath($path);
try {
$response = $this->client->propFind(
$this->encodePath($path),
['{DAV:}getetag'],
1
);
if ($response === false) {
return false;
}
$content = [];
$files = array_keys($response);
array_shift($files); //the first entry is the current directory

if (!$this->statCache->hasKey($path)) {
$this->statCache->set($path, true);
}
foreach ($files as $file) {
$file = urldecode($file);
// do not store the real entry, we might not have all properties
if (!$this->statCache->hasKey($path)) {
$this->statCache->set($file, true);
}
$file = basename($file);
$content[] = $file;
$content = $this->getDirectoryContent($path);
$files = [];
foreach ($content as $child) {
$files[] = $child['name'];
}
return IteratorDirectory::wrap($content);
return IteratorDirectory::wrap($files);
} catch (\Exception $e) {
$this->convertException($e, $path);
}
Expand Down Expand Up @@ -291,16 +286,7 @@ protected function propfind($path) {
try {
$response = $this->client->propFind(
$this->encodePath($path),
[
'{DAV:}getlastmodified',
'{DAV:}getcontentlength',
'{DAV:}getcontenttype',
'{http://owncloud.org/ns}permissions',
'{http://open-collaboration-services.org/ns}share-permissions',
'{DAV:}resourcetype',
'{DAV:}getetag',
'{DAV:}quota-available-bytes',
]
self::PROPFIND_PROPS
);
$this->statCache->set($path, $response);
} catch (ClientHttpException $e) {
Expand Down Expand Up @@ -607,53 +593,84 @@ public function copy($source, $target) {
return false;
}

/** {@inheritdoc} */
public function stat($path) {
try {
$response = $this->propfind($path);
if (!$response) {
return false;
public function getMetaData($path) {
if (Filesystem::isFileBlacklisted($path)) {
throw new ForbiddenException('Invalid path: ' . $path, false);
}
$response = $this->propfind($path);
if (!$response) {
return null;
} else {
return $this->getMetaFromPropfind($path, $response);
}
}
private function getMetaFromPropfind(string $path, array $response): array {
if (isset($response['{DAV:}getetag'])) {
$etag = trim($response['{DAV:}getetag'], '"');
if (strlen($etag) > 40) {
$etag = md5($etag);
}
return [
'mtime' => isset($response['{DAV:}getlastmodified']) ? strtotime($response['{DAV:}getlastmodified']) : null,
'size' => Util::numericToNumber($response['{DAV:}getcontentlength'] ?? 0),
];
} catch (\Exception $e) {
$this->convertException($e, $path);
} else {
$etag = parent::getETag($path);
}

$responseType = [];
if (isset($response["{DAV:}resourcetype"])) {
/** @var ResourceType[] $response */
$responseType = $response["{DAV:}resourcetype"]->getValue();
}
$type = (count($responseType) > 0 and $responseType[0] == "{DAV:}collection") ? 'dir' : 'file';
if ($type === 'dir') {
$mimeType = 'httpd/unix-directory';
} elseif (isset($response['{DAV:}getcontenttype'])) {
$mimeType = $response['{DAV:}getcontenttype'];
} else {
$mimeType = $this->mimeTypeDetector->detectPath($path);
}
return [];

if (isset($response['{http://owncloud.org/ns}permissions'])) {
$permissions = $this->parsePermissions($response['{http://owncloud.org/ns}permissions']);
} elseif ($type === 'dir') {
$permissions = Constants::PERMISSION_ALL;
} else {
$permissions = Constants::PERMISSION_ALL - Constants::PERMISSION_CREATE;
}

$mtime = isset($response['{DAV:}getlastmodified']) ? strtotime($response['{DAV:}getlastmodified']) : null;

if ($type === 'dir') {
$size = -1;
} else {
$size = Util::numericToNumber($response['{DAV:}getcontentlength'] ?? 0);
}

return [
'name' => basename($path),
'mtime' => $mtime,
'storage_mtime' => $mtime,
'size' => $size,
'permissions' => $permissions,
'etag' => $etag,
'mimetype' => $mimeType,
];
}

/** {@inheritdoc} */
public function getMimeType($path) {
$remoteMimetype = $this->getMimeTypeFromRemote($path);
if ($remoteMimetype === 'application/octet-stream') {
return \OC::$server->getMimeTypeDetector()->detectPath($path);
public function stat($path) {
$meta = $this->getMetaData($path);
if (!$meta) {
return false;
} else {
return $remoteMimetype;
return $meta;
}
}

public function getMimeTypeFromRemote($path) {
try {
$response = $this->propfind($path);
if ($response === false) {
return false;
}
$responseType = [];
if (isset($response["{DAV:}resourcetype"])) {
/** @var ResourceType[] $response */
$responseType = $response["{DAV:}resourcetype"]->getValue();
}
$type = (count($responseType) > 0 and $responseType[0] == "{DAV:}collection") ? 'dir' : 'file';
if ($type == 'dir') {
return 'httpd/unix-directory';
} elseif (isset($response['{DAV:}getcontenttype'])) {
return $response['{DAV:}getcontenttype'];
} else {
return 'application/octet-stream';
}
} catch (\Exception $e) {
/** {@inheritdoc} */
public function getMimeType($path) {
$meta = $this->getMetaData($path);
if ($meta) {
return $meta['mimetype'];
} else {
return false;
}
}
Expand Down Expand Up @@ -739,39 +756,22 @@ public function isDeletable($path) {

/** {@inheritdoc} */
public function getPermissions($path) {
$this->init();
$path = $this->cleanPath($path);
$response = $this->propfind($path);
if ($response === false) {
return 0;
}
if (isset($response['{http://owncloud.org/ns}permissions'])) {
return $this->parsePermissions($response['{http://owncloud.org/ns}permissions']);
} elseif ($this->is_dir($path)) {
return Constants::PERMISSION_ALL;
} elseif ($this->file_exists($path)) {
return Constants::PERMISSION_ALL - Constants::PERMISSION_CREATE;
$stat = $this->getMetaData($path);
if ($stat) {
return $stat['permissions'];
} else {
return 0;
}
}

/** {@inheritdoc} */
public function getETag($path) {
$this->init();
$path = $this->cleanPath($path);
$response = $this->propfind($path);
if ($response === false) {
$meta = $this->getMetaData($path);
if ($meta) {
return $meta['etag'];
} else {
return null;
}
if (isset($response['{DAV:}getetag'])) {
$etag = trim($response['{DAV:}getetag'], '"');
if (strlen($etag) > 40) {
$etag = md5($etag);
}
return $etag;
}
return parent::getEtag($path);
}

/**
Expand Down Expand Up @@ -901,4 +901,33 @@ protected function convertException(Exception $e, $path = '') {

// TODO: only log for now, but in the future need to wrap/rethrow exception
}

public function getDirectoryContent($directory): \Traversable {
$this->init();
$directory = $this->cleanPath($directory);
try {
$responses = $this->client->propFind(
$this->encodePath($directory),
self::PROPFIND_PROPS,
1
);
if ($responses === false) {
return;
}

array_shift($responses); //the first entry is the current directory
if (!$this->statCache->hasKey($directory)) {
$this->statCache->set($directory, true);
}

foreach ($responses as $file => $response) {
$file = urldecode($file);
$file = substr($file, strlen($this->root));
$this->statCache->set($file, $response);
yield $this->getMetaFromPropfind($file, $response);
}
} catch (\Exception $e) {
$this->convertException($e, $directory);
}
}
}