Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
63 commits
Select commit Hold shift + click to select a range
00894e2
feat: first pass at TaskProcessing API
marcelklehr Apr 29, 2024
8352b27
fix: weed out some psalm errors and run cs:fix
marcelklehr Apr 30, 2024
4b2acee
test: Add OldTasksShouldBeCleanedUp test
marcelklehr Apr 30, 2024
17486ad
fix: Add RemoveOldTasksBackgroundJob to repair step that instantiates it
marcelklehr Apr 30, 2024
ee7592f
fix: Run cs:fix
marcelklehr Apr 30, 2024
44b896f
feat: TaskProcessing OCS API
marcelklehr Apr 30, 2024
b2b93e4
feat: Add getFileContents endpoint to TaskProcessing OCS API
marcelklehr May 2, 2024
86317bb
refactor: Move validation to EShapeType
marcelklehr May 2, 2024
29cbb3c
chore: Run cs:fix
marcelklehr May 2, 2024
3b0925a
chore: Regenerate openapi.json
marcelklehr May 2, 2024
1c033ae
fix(IRootFolder): Add getAppDataDirectoryName method
marcelklehr May 2, 2024
7a94798
fix: Fix psalm issues
marcelklehr May 2, 2024
8e56626
feat: Add ExApp endpoints
marcelklehr May 3, 2024
5031a2e
fix: Typo
marcelklehr May 3, 2024
843bb62
fix: LazyRoot missing method
marcelklehr May 3, 2024
c9ea537
fix: Fix psalm issues
marcelklehr May 3, 2024
b150d77
refactor: rename getTaskType to getTaskTypeId
marcelklehr May 3, 2024
4b954d2
fix: Wire-up the new classes
marcelklehr May 3, 2024
eebeb82
fix: Small fixes
marcelklehr May 3, 2024
bd5dfd0
test: Add more tests for legacy pass-through
marcelklehr May 3, 2024
3593d9b
fix: psalm issue
marcelklehr May 3, 2024
9a2cd6b
fix: Expose task type on CoreTaskProcessingTask json
marcelklehr May 6, 2024
928d04f
fix: oc_taskProcessing_tasks.identifier: notnull = false
marcelklehr May 6, 2024
9effb55
chore: update openapi.json
marcelklehr May 6, 2024
996e507
feat: Remove stale files in AppData as well
marcelklehr May 6, 2024
6203c1c
fix: Check if user is authorized to use the files they mentioned
marcelklehr May 6, 2024
ef44af1
test: Test file authorization check
marcelklehr May 6, 2024
8ccb29a
fix: psalm issues
marcelklehr May 6, 2024
a5053d3
fix: Run cs:fix
marcelklehr May 6, 2024
b85a0ed
fix: Update autoloaders
marcelklehr May 6, 2024
5de42a5
fix: Don't use dynamic property
marcelklehr May 6, 2024
2c87809
fix: address review comments
marcelklehr May 6, 2024
ef61c50
fix: address review comments
marcelklehr May 6, 2024
ec27c53
fix: address review comments
marcelklehr May 6, 2024
fff2fb8
fix: psalm issue
marcelklehr May 7, 2024
4a3b9b8
refactor: identifier is now customId/custom_id
marcelklehr May 7, 2024
20c09d1
fix: Don't check in barmani plugin in composer autoloader
marcelklehr May 7, 2024
beeee26
fix: bump OC_Version
marcelklehr May 7, 2024
4d9a0ea
fix: update openai specs
marcelklehr May 7, 2024
f2ab6cb
fix: fix psalm issues
marcelklehr May 7, 2024
c079a61
feat: Add cancel endpoint to OCS API
marcelklehr May 7, 2024
a045e0c
fix: fix migration
marcelklehr May 7, 2024
0e06d64
Update core/Migrations/Version30000Date20240429122720.php
marcelklehr May 7, 2024
f6f4965
fix: fix tests
marcelklehr May 7, 2024
eff862b
fix: run cs:fix
marcelklehr May 8, 2024
ec94a67
fix(ocs): change /tasktypes response to combine optional and non-opti…
marcelklehr May 8, 2024
19a0aae
fix(TextToImage): Allow leaving the resources open
marcelklehr May 10, 2024
f3a88f0
fix(OCS-API): No csrf required for /tasks/taskId/file/fileId
marcelklehr May 10, 2024
a8afa7f
fix(OCS-API): Add endpoint to list user tasks
marcelklehr May 10, 2024
9cc1a01
test: Put input files in user storage
marcelklehr May 10, 2024
715245a
fix: run cs:fix
marcelklehr May 10, 2024
e4b1d23
fix(Manager#fillInputData): Load user folder if needed
marcelklehr May 10, 2024
c020490
Update lib/private/TaskProcessing/Manager.php
marcelklehr May 13, 2024
b11052f
Update lib/public/TaskProcessing/IManager.php
marcelklehr May 13, 2024
ac36c78
fix(SynchronousBackgroundJob): Only reschedule when needed
marcelklehr May 13, 2024
a9a2cbf
feat: Add some new task types
marcelklehr May 13, 2024
c1f84aa
fix(Manager#fillInputs): Try to setup user FS before access file inputs
marcelklehr May 13, 2024
4ce5aaf
chore: Check in autoloader updates
marcelklehr May 13, 2024
9864fc8
chore: fix htaccess
marcelklehr May 13, 2024
8d06338
fix: Fix pass-through stt provider
marcelklehr May 13, 2024
f5a8bda
Update core/ResponseDefinitions.php
marcelklehr May 14, 2024
cac812d
fix: address review comments
marcelklehr May 14, 2024
6c4992d
fix: expose lastUpdated in OCS API
marcelklehr May 15, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
refactor: identifier is now customId/custom_id
Signed-off-by: Marcel Klehr <[email protected]>
  • Loading branch information
marcelklehr committed May 14, 2024
commit 4a3b9b826ea532991f8636b621f92760c321e93e
4 changes: 2 additions & 2 deletions core/Controller/TaskProcessingApiController.php
Original file line number Diff line number Diff line change
Expand Up @@ -114,8 +114,8 @@ public function taskTypes(): DataResponse {
#[UserRateLimit(limit: 20, period: 120)]
#[AnonRateLimit(limit: 5, period: 120)]
#[ApiRoute(verb: 'POST', url: '/schedule', root: '/taskprocessing')]
public function schedule(array $input, string $type, string $appId, string $identifier = ''): DataResponse {
$task = new Task($type, $input, $appId, $this->userId, $identifier);
public function schedule(array $input, string $type, string $appId, string $customId = ''): DataResponse {
$task = new Task($type, $input, $appId, $this->userId, $customId);
try {
$this->taskProcessingManager->scheduleTask($task);

Expand Down
2 changes: 1 addition & 1 deletion core/ResponseDefinitions.php
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@
* appId: string,
* input: array<string, numeric|list<numeric>|string|list<string>>,
* output: ?array<string, numeric|list<numeric>|string|list<string>>,
* identifier: ?string,
* customId: ?string,
* completionExpectedAt: ?int,
* progress: ?float
* }
Expand Down
37 changes: 18 additions & 19 deletions lib/private/Log/PsrLoggerAdapter.php
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@
use OCP\Log\IDataLogger;
use Psr\Log\InvalidArgumentException;
use Psr\Log\LoggerInterface;
use Stringable;
use Throwable;
use function array_key_exists;
use function array_merge;
Expand All @@ -53,10 +52,10 @@ private function containsThrowable(array $context): bool {
/**
* System is unusable.
*
* @param string|Stringable $message
* @param $message
* @param mixed[] $context
*/
public function emergency(string|Stringable $message, array $context = []): void {
public function emergency($message, array $context = []): void {
if ($this->containsThrowable($context)) {
$this->logger->logException($context['exception'], array_merge(
[
Expand All @@ -76,10 +75,10 @@ public function emergency(string|Stringable $message, array $context = []): void
* Example: Entire website down, database unavailable, etc. This should
* trigger the SMS alerts and wake you up.
*
* @param string|Stringable $message
* @param $message
* @param mixed[] $context
*/
public function alert(string|Stringable $message, array $context = []): void {
public function alert($message, array $context = []): void {
if ($this->containsThrowable($context)) {
$this->logger->logException($context['exception'], array_merge(
[
Expand All @@ -98,10 +97,10 @@ public function alert(string|Stringable $message, array $context = []): void {
*
* Example: Application component unavailable, unexpected exception.
*
* @param string|Stringable $message
* @param $message
* @param mixed[] $context
*/
public function critical(string|Stringable $message, array $context = []): void {
public function critical($message, array $context = []): void {
if ($this->containsThrowable($context)) {
$this->logger->logException($context['exception'], array_merge(
[
Expand All @@ -119,10 +118,10 @@ public function critical(string|Stringable $message, array $context = []): void
* Runtime errors that do not require immediate action but should typically
* be logged and monitored.
*
* @param string|Stringable $message
* @param $message
* @param mixed[] $context
*/
public function error(string|Stringable $message, array $context = []): void {
public function error($message, array $context = []): void {
if ($this->containsThrowable($context)) {
$this->logger->logException($context['exception'], array_merge(
[
Expand All @@ -142,10 +141,10 @@ public function error(string|Stringable $message, array $context = []): void {
* Example: Use of deprecated APIs, poor use of an API, undesirable things
* that are not necessarily wrong.
*
* @param string|Stringable $message
* @param $message
* @param mixed[] $context
*/
public function warning(string|Stringable $message, array $context = []): void {
public function warning($message, array $context = []): void {
if ($this->containsThrowable($context)) {
$this->logger->logException($context['exception'], array_merge(
[
Expand All @@ -162,10 +161,10 @@ public function warning(string|Stringable $message, array $context = []): void {
/**
* Normal but significant events.
*
* @param string|Stringable $message
* @param $message
* @param mixed[] $context
*/
public function notice(string|Stringable $message, array $context = []): void {
public function notice($message, array $context = []): void {
if ($this->containsThrowable($context)) {
$this->logger->logException($context['exception'], array_merge(
[
Expand All @@ -184,10 +183,10 @@ public function notice(string|Stringable $message, array $context = []): void {
*
* Example: User logs in, SQL logs.
*
* @param string|Stringable $message
* @param $message
* @param mixed[] $context
*/
public function info(string|Stringable $message, array $context = []): void {
public function info($message, array $context = []): void {
if ($this->containsThrowable($context)) {
$this->logger->logException($context['exception'], array_merge(
[
Expand All @@ -204,10 +203,10 @@ public function info(string|Stringable $message, array $context = []): void {
/**
* Detailed debug information.
*
* @param string|Stringable $message
* @param $message
* @param mixed[] $context
*/
public function debug(string|Stringable $message, array $context = []): void {
public function debug($message, array $context = []): void {
if ($this->containsThrowable($context)) {
$this->logger->logException($context['exception'], array_merge(
[
Expand All @@ -225,12 +224,12 @@ public function debug(string|Stringable $message, array $context = []): void {
* Logs with an arbitrary level.
*
* @param mixed $level
* @param string|Stringable $message
* @param $message
* @param mixed[] $context
*
* @throws InvalidArgumentException
*/
public function log($level, string|Stringable $message, array $context = []): void {
public function log($level, $message, array $context = []): void {
if (!is_int($level) || $level < ILogger::DEBUG || $level > ILogger::FATAL) {
throw new InvalidArgumentException('Nextcloud allows only integer log levels');
}
Expand Down
12 changes: 6 additions & 6 deletions lib/private/TaskProcessing/Db/Task.php
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@
* @method string|null getUserId()
* @method setAppId(string $type)
* @method string getAppId()
* @method setIdentifier(string $identifier)
* @method setIdentifier(string $customId)
* @method string getIdentifier()
* @method setCompletionExpectedAt(null|\DateTime $completionExpectedAt)
* @method null|\DateTime getCompletionExpectedAt()
Expand All @@ -60,20 +60,20 @@ class Task extends Entity {
protected $status;
protected $userId;
protected $appId;
protected $identifier;
protected $customId;
protected $completionExpectedAt;
protected $errorMessage;
protected $progress;

/**
* @var string[]
*/
public static array $columns = ['id', 'last_updated', 'type', 'input', 'output', 'status', 'user_id', 'app_id', 'identifier', 'completion_expected_at', 'error_message', 'progress'];
public static array $columns = ['id', 'last_updated', 'type', 'input', 'output', 'status', 'user_id', 'app_id', 'custom_id', 'completion_expected_at', 'error_message', 'progress'];

/**
* @var string[]
*/
public static array $fields = ['id', 'lastUpdated', 'type', 'input', 'output', 'status', 'userId', 'appId', 'identifier', 'completionExpectedAt', 'errorMessage', 'progress'];
public static array $fields = ['id', 'lastUpdated', 'type', 'input', 'output', 'status', 'userId', 'appId', 'customId', 'completionExpectedAt', 'errorMessage', 'progress'];


public function __construct() {
Expand All @@ -86,7 +86,7 @@ public function __construct() {
$this->addType('status', 'integer');
$this->addType('userId', 'string');
$this->addType('appId', 'string');
$this->addType('identifier', 'string');
$this->addType('customId', 'string');
$this->addType('completionExpectedAt', 'datetime');
$this->addType('errorMessage', 'string');
$this->addType('progress', 'float');
Expand All @@ -110,7 +110,7 @@ public static function fromPublicTask(OCPTask $task): self {
'errorMessage' => $task->getErrorMessage(),
'userId' => $task->getUserId(),
'appId' => $task->getAppId(),
'identifier' => $task->getIdentifier(),
'customId' => $task->getIdentifier(),
'completionExpectedAt' => $task->getCompletionExpectedAt(),
'progress' => $task->getProgress(),
]);
Expand Down
8 changes: 4 additions & 4 deletions lib/private/TaskProcessing/Db/TaskMapper.php
Original file line number Diff line number Diff line change
Expand Up @@ -103,18 +103,18 @@ public function findByIdAndUser(int $id, ?string $userId): Task {
/**
* @param string $userId
* @param string $appId
* @param string|null $identifier
* @param string|null $customId
* @return list<Task>
* @throws Exception
*/
public function findUserTasksByApp(string $userId, string $appId, ?string $identifier = null): array {
public function findUserTasksByApp(string $userId, string $appId, ?string $customId = null): array {
$qb = $this->db->getQueryBuilder();
$qb->select(Task::$columns)
->from($this->tableName)
->where($qb->expr()->eq('user_id', $qb->createPositionalParameter($userId)))
->andWhere($qb->expr()->eq('app_id', $qb->createPositionalParameter($appId)));
if ($identifier !== null) {
$qb->andWhere($qb->expr()->eq('identifier', $qb->createPositionalParameter($identifier)));
if ($customId !== null) {
$qb->andWhere($qb->expr()->eq('custom_id', $qb->createPositionalParameter($customId)));
}
return array_values($this->findEntities($qb));
}
Expand Down
4 changes: 2 additions & 2 deletions lib/private/TaskProcessing/Manager.php
Original file line number Diff line number Diff line change
Expand Up @@ -797,9 +797,9 @@ public function getUserTask(int $id, ?string $userId): Task {
}
}

public function getUserTasksByApp(?string $userId, string $appId, ?string $identifier = null): array {
public function getUserTasksByApp(?string $userId, string $appId, ?string $customId = null): array {
try {
$taskEntities = $this->taskMapper->findUserTasksByApp($userId, $appId, $identifier);
$taskEntities = $this->taskMapper->findUserTasksByApp($userId, $appId, $customId);
return array_map(fn ($taskEntity): Task => $taskEntity->toPublicTask(), $taskEntities);
} catch (\OCP\DB\Exception $e) {
throw new \OCP\TaskProcessing\Exception\Exception('There was a problem finding a task', 0, $e);
Expand Down
2 changes: 1 addition & 1 deletion lib/private/TaskProcessing/SynchronousBackgroundJob.php
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ protected function run($argument) {
return;
}
try {
$output = $provider->process($task->getUserId(), $input);
$output = $provider->process($task->getUserId(), $input, fn(float $progress) => $this->taskProcessingManager->setTaskProgress($task->getId(), $progress));
} catch (ProcessingException $e) {
$this->logger->warning('Failed to process a TaskProcessing task with synchronous provider ' . $provider->getId(), ['exception' => $e]);
$this->taskProcessingManager->setTaskResult($task->getId(), $e->getMessage(), null);
Expand Down
4 changes: 2 additions & 2 deletions lib/public/TaskProcessing/IManager.php
Original file line number Diff line number Diff line change
Expand Up @@ -138,13 +138,13 @@ public function getUserTask(int $id, ?string $userId): Task;
/**
* @param string|null $userId
* @param string $appId
* @param string|null $identifier
* @param string|null $customId
* @return list<Task>
* @throws Exception If the query failed
* @throws \JsonException If parsing the task input and output failed
* @since 30.0.0
*/
public function getUserTasksByApp(?string $userId, string $appId, ?string $identifier = null): array;
public function getUserTasksByApp(?string $userId, string $appId, ?string $customId = null): array;

/**
* Prepare the task's input data, so it can be processed by the provider
Expand Down
3 changes: 2 additions & 1 deletion lib/public/TaskProcessing/ISynchronousProvider.php
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,10 @@ interface ISynchronousProvider extends IProvider {
*
* @param null|string $userId The user that created the current task
* @param array<string, list<numeric|string|File>|numeric|string|File> $input The task input
* @param callable(float):bool $reportProgress Report the task progress. If this returns false, that means the task was cancelled and processing should be stopped.
* @psalm-return array<string, list<numeric|string>|numeric|string>
* @throws ProcessingException
*@since 30.0.0
*/
public function process(?string $userId, array $input): array;
public function process(?string $userId, array $input, callable $reportProgress): array;
}
12 changes: 6 additions & 6 deletions lib/public/TaskProcessing/Task.php
Original file line number Diff line number Diff line change
Expand Up @@ -79,15 +79,15 @@ final class Task implements \JsonSerializable {
* @param array<string,list<numeric|string>|numeric|string> $input
* @param string $appId
* @param string|null $userId
* @param null|string $identifier An arbitrary identifier for this task. max length: 255 chars
* @param null|string $customId An arbitrary customId for this task. max length: 255 chars
* @since 30.0.0
*/
final public function __construct(
protected readonly string $taskTypeId,
protected array $input,
protected readonly string $appId,
protected readonly ?string $userId,
protected readonly ?string $identifier = '',
protected readonly ?string $customId = '',
) {
}

Expand Down Expand Up @@ -182,8 +182,8 @@ final public function getAppId(): string {
* @return null|string
* @since 30.0.0
*/
final public function getIdentifier(): ?string {
return $this->identifier;
final public function getCustomId(): ?string {
return $this->customId;
}

/**
Expand All @@ -195,7 +195,7 @@ final public function getUserId(): ?string {
}

/**
* @psalm-return array{id: ?int, type: string, status: 'STATUS_CANCELLED'|'STATUS_FAILED'|'STATUS_SUCCESSFUL'|'STATUS_RUNNING'|'STATUS_SCHEDULED'|'STATUS_UNKNOWN', userId: ?string, appId: string, input: array<array-key, list<numeric|string>|numeric|string>, output: ?array<array-key, list<numeric|string>|numeric|string>, identifier: ?string, completionExpectedAt: ?int, progress: ?float}
* @psalm-return array{id: ?int, type: string, status: 'STATUS_CANCELLED'|'STATUS_FAILED'|'STATUS_SUCCESSFUL'|'STATUS_RUNNING'|'STATUS_SCHEDULED'|'STATUS_UNKNOWN', userId: ?string, appId: string, input: array<array-key, list<numeric|string>|numeric|string>, output: ?array<array-key, list<numeric|string>|numeric|string>, customId: ?string, completionExpectedAt: ?int, progress: ?float}
* @since 30.0.0
*/
final public function jsonSerialize(): array {
Expand All @@ -207,7 +207,7 @@ final public function jsonSerialize(): array {
'appId' => $this->getAppId(),
'input' => $this->getInput(),
'output' => $this->getOutput(),
'identifier' => $this->getIdentifier(),
'customId' => $this->getCustomId(),
'completionExpectedAt' => $this->getCompletionExpectedAt()?->getTimestamp(),
'progress' => $this->getProgress(),
];
Expand Down