feat(taskprocessing): add cleanup flag to tasks

core/Command/TaskProcessing/Cleanup.php

@@ -0,0 +1,93 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+namespace OC\Core\Command\TaskProcessing;
+
+use OC\Core\Command\Base;
+use OC\TaskProcessing\Db\TaskMapper;
+use OC\TaskProcessing\Manager;
+use OCP\Files\AppData\IAppDataFactory;
+use Psr\Log\LoggerInterface;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+
+class Cleanup extends Base {
+	private \OCP\Files\IAppData $appData;
+
+	public function __construct(
+		protected Manager $taskProcessingManager,
+		private TaskMapper $taskMapper,
+		private LoggerInterface $logger,
+		IAppDataFactory $appDataFactory,
+	) {
+		parent::__construct();
+		$this->appData = $appDataFactory->get('core');
+	}
+
+	protected function configure() {
+		$this
+			->setName('taskprocessing:task:cleanup')
+			->setDescription('cleanup old tasks')
+			->addArgument(
+				'maxAgeSeconds',
+				InputArgument::OPTIONAL,
+				// default is not defined as an argument default value because we want to show a nice "4 months" value
+				'delete tasks that are older than this number of seconds, defaults to ' . Manager::MAX_TASK_AGE_SECONDS . ' (4 months)',
+			);
+		parent::configure();
+	}
+
+	protected function execute(InputInterface $input, OutputInterface $output): int {
+		$maxAgeSeconds = $input->getArgument('maxAgeSeconds') ?? Manager::MAX_TASK_AGE_SECONDS;
+		$output->writeln('<comment>Cleanup up tasks older than ' . $maxAgeSeconds . ' seconds and the related output files</comment>');
+
+		$taskIdsToCleanup = [];
+		try {
+			$fileCleanupGenerator = $this->taskProcessingManager->cleanupTaskProcessingTaskFiles($maxAgeSeconds);
+			foreach ($fileCleanupGenerator as $cleanedUpEntry) {
+				$output->writeln(
+					"<info>\t - " . 'Deleted appData/core/TaskProcessing/' . $cleanedUpEntry['file_name']
+					. ' (fileId: ' . $cleanedUpEntry['file_id'] . ', taskId: ' . $cleanedUpEntry['task_id'] . ')</info>'
+				);
+			}
+			$taskIdsToCleanup = $fileCleanupGenerator->getReturn();
+		} catch (\Exception $e) {
+			$this->logger->warning('Failed to delete stale task processing tasks files', ['exception' => $e]);
+			$output->writeln('<warning>Failed to delete stale task processing tasks files</warning>');
+		}
+		try {
+			$deletedTaskCount = $this->taskMapper->deleteOlderThan($maxAgeSeconds);
+			foreach ($taskIdsToCleanup as $taskId) {
+				$output->writeln("<info>\t - " . 'Deleted task ' . $taskId . ' from the database</info>');
+			}
+			$output->writeln("<comment>\t - " . 'Deleted ' . $deletedTaskCount . ' tasks from the database</comment>');
+		} catch (\OCP\DB\Exception $e) {
+			$this->logger->warning('Failed to delete stale task processing tasks', ['exception' => $e]);
+			$output->writeln('<warning>Failed to delete stale task processing tasks</warning>');
+		}
+		try {
+			$textToImageDeletedFileNames = $this->taskProcessingManager->clearFilesOlderThan($this->appData->getFolder('text2image'), $maxAgeSeconds);
+			foreach ($textToImageDeletedFileNames as $entry) {
+				$output->writeln("<info>\t - " . 'Deleted appData/core/text2image/' . $entry . '</info>');
+			}
+		} catch (\OCP\Files\NotFoundException $e) {
+			// noop
+		}
+		try {
+			$audioToTextDeletedFileNames = $this->taskProcessingManager->clearFilesOlderThan($this->appData->getFolder('audio2text'), $maxAgeSeconds);
+			foreach ($audioToTextDeletedFileNames as $entry) {
+				$output->writeln("<info>\t - " . 'Deleted appData/core/audio2text/' . $entry . '</info>');
+			}
+		} catch (\OCP\Files\NotFoundException $e) {
+			// noop
+		}
+
+		return 0;
+	}
+}

core/Command/TaskProcessing/EnabledCommand.php

@@ -1,5 +1,7 @@
 <?php
 
+declare(strict_types=1);
+
 /**
  * SPDX-FileCopyrightText: 2021 Nextcloud GmbH and Nextcloud contributors
  * SPDX-License-Identifier: AGPL-3.0-or-later

core/Command/TaskProcessing/GetCommand.php

@@ -1,5 +1,7 @@
 <?php
 
+declare(strict_types=1);
+
 /**
  * SPDX-FileCopyrightText: 2021 Nextcloud GmbH and Nextcloud contributors
  * SPDX-License-Identifier: AGPL-3.0-or-later

core/Command/TaskProcessing/ListCommand.php

@@ -1,5 +1,7 @@
 <?php
 
+declare(strict_types=1);
+
 /**
  * SPDX-FileCopyrightText: 2021 Nextcloud GmbH and Nextcloud contributors
  * SPDX-License-Identifier: AGPL-3.0-or-later

core/Command/TaskProcessing/Statistics.php

@@ -1,5 +1,7 @@
 <?php
 
+declare(strict_types=1);
+
 /**
  * SPDX-FileCopyrightText: 2021 Nextcloud GmbH and Nextcloud contributors
  * SPDX-License-Identifier: AGPL-3.0-or-later

core/Controller/TaskProcessingApiController.php

@@ -31,7 +31,6 @@
 use OCP\IL10N;
 use OCP\IRequest;
 use OCP\Lock\LockedException;
-use OCP\TaskProcessing\EShapeType;
 use OCP\TaskProcessing\Exception\Exception;
 use OCP\TaskProcessing\Exception\NotFoundException;
 use OCP\TaskProcessing\Exception\PreConditionNotMetException;
@@ -391,7 +390,7 @@ public function setFileContentsExApp(int $taskId): DataResponse {
 	 * @return StreamResponse<Http::STATUS_OK, array{}>|DataResponse<Http::STATUS_INTERNAL_SERVER_ERROR|Http::STATUS_NOT_FOUND, array{message: string}, array{}>
 	 */
 	private function getFileContentsInternal(Task $task, int $fileId): StreamResponse|DataResponse {
-		$ids = $this->extractFileIdsFromTask($task);
+		$ids = $this->taskProcessingManager->extractFileIdsFromTask($task);
 		if (!in_array($fileId, $ids)) {
 			return new DataResponse(['message' => $this->l->t('Not found')], Http::STATUS_NOT_FOUND);
 		}
@@ -428,45 +427,6 @@ private function getFileContentsInternal(Task $task, int $fileId): StreamRespons
 		return $response;
 	}
 
-	/**
-	 * @param Task $task
-	 * @return list<int>
-	 * @throws NotFoundException
-	 */
-	private function extractFileIdsFromTask(Task $task): array {
-		$ids = [];
-		$taskTypes = $this->taskProcessingManager->getAvailableTaskTypes();
-		if (!isset($taskTypes[$task->getTaskTypeId()])) {
-			throw new NotFoundException('Could not find task type');
-		}
-		$taskType = $taskTypes[$task->getTaskTypeId()];
-		foreach ($taskType['inputShape'] + $taskType['optionalInputShape'] as $key => $descriptor) {
-			if (in_array(EShapeType::getScalarType($descriptor->getShapeType()), [EShapeType::File, EShapeType::Image, EShapeType::Audio, EShapeType::Video], true)) {
-				/** @var int|list<int> $inputSlot */
-				$inputSlot = $task->getInput()[$key];
-				if (is_array($inputSlot)) {
-					$ids = array_merge($inputSlot, $ids);
-				} else {
-					$ids[] = $inputSlot;
-				}
-			}
-		}
-		if ($task->getOutput() !== null) {
-			foreach ($taskType['outputShape'] + $taskType['optionalOutputShape'] as $key => $descriptor) {
-				if (in_array(EShapeType::getScalarType($descriptor->getShapeType()), [EShapeType::File, EShapeType::Image, EShapeType::Audio, EShapeType::Video], true)) {
-					/** @var int|list<int> $outputSlot */
-					$outputSlot = $task->getOutput()[$key];
-					if (is_array($outputSlot)) {
-						$ids = array_merge($outputSlot, $ids);
-					} else {
-						$ids[] = $outputSlot;
-					}
-				}
-			}
-		}
-		return $ids;
-	}
-
 	/**
 	 * Sets the task progress
 	 *

core/Migrations/Version32000Date20250806110519.php

@@ -0,0 +1,49 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+namespace OC\Core\Migrations;
+
+use Closure;
+use OCP\DB\ISchemaWrapper;
+use OCP\DB\Types;
+use OCP\Migration\Attributes\AddColumn;
+use OCP\Migration\Attributes\ColumnType;
+use OCP\Migration\IOutput;
+use OCP\Migration\SimpleMigrationStep;
+
+/**
+ *
+ */
+#[AddColumn(table: 'taskprocessing_tasks', name: 'allow_cleanup', type: ColumnType::SMALLINT)]
+class Version32000Date20250806110519 extends SimpleMigrationStep {
+
+	/**
+	 * @param IOutput $output
+	 * @param Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper`
+	 * @param array $options
+	 * @return null|ISchemaWrapper
+	 */
+	public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
+		/** @var ISchemaWrapper $schema */
+		$schema = $schemaClosure();
+
+		if ($schema->hasTable('taskprocessing_tasks')) {
+			$table = $schema->getTable('taskprocessing_tasks');
+			if (!$table->hasColumn('allow_cleanup')) {
+				$table->addColumn('allow_cleanup', Types::SMALLINT, [
+					'notnull' => true,
+					'default' => 1,
+					'unsigned' => true,
+				]);
+				return $schema;
+			}
+		}
+
+		return null;
+	}
+}

core/ResponseDefinitions.php

@@ -200,6 +200,7 @@
  *     scheduledAt: ?int,
  *     startedAt: ?int,
  *     endedAt: ?int,
+ *     allowCleanup: bool,
  * }
  *
  * @psalm-type CoreProfileAction = array{

core/openapi-ex_app.json

@@ -145,7 +145,8 @@
                     "progress",
                     "scheduledAt",
                     "startedAt",
-                    "endedAt"
+                    "endedAt",
+                    "allowCleanup"
                 ],
                 "properties": {
                     "id": {
@@ -216,6 +217,9 @@
                         "type": "integer",
                         "format": "int64",
                         "nullable": true
+                    },
+                    "allowCleanup": {
+                        "type": "boolean"
                     }
                 }
             }

core/openapi-full.json

@@ -639,7 +639,8 @@
                     "progress",
                     "scheduledAt",
                     "startedAt",
-                    "endedAt"
+                    "endedAt",
+                    "allowCleanup"
                 ],
                 "properties": {
                     "id": {
@@ -710,6 +711,9 @@
                         "type": "integer",
                         "format": "int64",
                         "nullable": true
+                    },
+                    "allowCleanup": {
+                        "type": "boolean"
                     }
                 }
             },

core/openapi.json

@@ -639,7 +639,8 @@
                     "progress",
                     "scheduledAt",
                     "startedAt",
-                    "endedAt"
+                    "endedAt",
+                    "allowCleanup"
                 ],
                 "properties": {
                     "id": {
@@ -710,6 +711,9 @@
                         "type": "integer",
                         "format": "int64",
                         "nullable": true
+                    },
+                    "allowCleanup": {
+                        "type": "boolean"
                     }
                 }
             },

core/register_command.php

@@ -253,6 +253,7 @@
 	$application->add(Server::get(EnabledCommand::class));
 	$application->add(Server::get(Command\TaskProcessing\ListCommand::class));
 	$application->add(Server::get(Statistics::class));
+	$application->add(Server::get(Command\TaskProcessing\Cleanup::class));
 
 	$application->add(Server::get(RedisCommand::class));
 	$application->add(Server::get(DistributedClear::class));

lib/composer/composer/autoload_classmap.php

@@ -1346,6 +1346,7 @@
     'OC\\Core\\Command\\SystemTag\\Delete' => $baseDir . '/core/Command/SystemTag/Delete.php',
     'OC\\Core\\Command\\SystemTag\\Edit' => $baseDir . '/core/Command/SystemTag/Edit.php',
     'OC\\Core\\Command\\SystemTag\\ListCommand' => $baseDir . '/core/Command/SystemTag/ListCommand.php',
+    'OC\\Core\\Command\\TaskProcessing\\Cleanup' => $baseDir . '/core/Command/TaskProcessing/Cleanup.php',
     'OC\\Core\\Command\\TaskProcessing\\EnabledCommand' => $baseDir . '/core/Command/TaskProcessing/EnabledCommand.php',
     'OC\\Core\\Command\\TaskProcessing\\GetCommand' => $baseDir . '/core/Command/TaskProcessing/GetCommand.php',
     'OC\\Core\\Command\\TaskProcessing\\ListCommand' => $baseDir . '/core/Command/TaskProcessing/ListCommand.php',
@@ -1512,6 +1513,7 @@
     'OC\\Core\\Migrations\\Version31000Date20250213102442' => $baseDir . '/core/Migrations/Version31000Date20250213102442.php',
     'OC\\Core\\Migrations\\Version32000Date20250620081925' => $baseDir . '/core/Migrations/Version32000Date20250620081925.php',
     'OC\\Core\\Migrations\\Version32000Date20250731062008' => $baseDir . '/core/Migrations/Version32000Date20250731062008.php',
+    'OC\\Core\\Migrations\\Version32000Date20250806110519' => $baseDir . '/core/Migrations/Version32000Date20250806110519.php',
     'OC\\Core\\Notification\\CoreNotifier' => $baseDir . '/core/Notification/CoreNotifier.php',
     'OC\\Core\\ResponseDefinitions' => $baseDir . '/core/ResponseDefinitions.php',
     'OC\\Core\\Service\\LoginFlowV2Service' => $baseDir . '/core/Service/LoginFlowV2Service.php',

lib/composer/composer/autoload_static.php

@@ -1387,6 +1387,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
         'OC\\Core\\Command\\SystemTag\\Delete' => __DIR__ . '/../../..' . '/core/Command/SystemTag/Delete.php',
         'OC\\Core\\Command\\SystemTag\\Edit' => __DIR__ . '/../../..' . '/core/Command/SystemTag/Edit.php',
         'OC\\Core\\Command\\SystemTag\\ListCommand' => __DIR__ . '/../../..' . '/core/Command/SystemTag/ListCommand.php',
+        'OC\\Core\\Command\\TaskProcessing\\Cleanup' => __DIR__ . '/../../..' . '/core/Command/TaskProcessing/Cleanup.php',
         'OC\\Core\\Command\\TaskProcessing\\EnabledCommand' => __DIR__ . '/../../..' . '/core/Command/TaskProcessing/EnabledCommand.php',
         'OC\\Core\\Command\\TaskProcessing\\GetCommand' => __DIR__ . '/../../..' . '/core/Command/TaskProcessing/GetCommand.php',
         'OC\\Core\\Command\\TaskProcessing\\ListCommand' => __DIR__ . '/../../..' . '/core/Command/TaskProcessing/ListCommand.php',
@@ -1553,6 +1554,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
         'OC\\Core\\Migrations\\Version31000Date20250213102442' => __DIR__ . '/../../..' . '/core/Migrations/Version31000Date20250213102442.php',
         'OC\\Core\\Migrations\\Version32000Date20250620081925' => __DIR__ . '/../../..' . '/core/Migrations/Version32000Date20250620081925.php',
         'OC\\Core\\Migrations\\Version32000Date20250731062008' => __DIR__ . '/../../..' . '/core/Migrations/Version32000Date20250731062008.php',
+        'OC\\Core\\Migrations\\Version32000Date20250806110519' => __DIR__ . '/../../..' . '/core/Migrations/Version32000Date20250806110519.php',
         'OC\\Core\\Notification\\CoreNotifier' => __DIR__ . '/../../..' . '/core/Notification/CoreNotifier.php',
         'OC\\Core\\ResponseDefinitions' => __DIR__ . '/../../..' . '/core/ResponseDefinitions.php',
         'OC\\Core\\Service\\LoginFlowV2Service' => __DIR__ . '/../../..' . '/core/Service/LoginFlowV2Service.php',

lib/private/TaskProcessing/Db/Task.php

@@ -45,6 +45,8 @@
  * @method int getStartedAt()
  * @method setEndedAt(int $endedAt)
  * @method int getEndedAt()
+ * @method setAllowCleanup(int $allowCleanup)
+ * @method int getAllowCleanup()
  */
 class Task extends Entity {
 	protected $lastUpdated;
@@ -63,16 +65,17 @@ class Task extends Entity {
 	protected $scheduledAt;
 	protected $startedAt;
 	protected $endedAt;
+	protected $allowCleanup;
 
 	/**
 	 * @var string[]
 	 */
-	public static array $columns = ['id', 'last_updated', 'type', 'input', 'output', 'status', 'user_id', 'app_id', 'custom_id', 'completion_expected_at', 'error_message', 'progress', 'webhook_uri', 'webhook_method', 'scheduled_at', 'started_at', 'ended_at'];
+	public static array $columns = ['id', 'last_updated', 'type', 'input', 'output', 'status', 'user_id', 'app_id', 'custom_id', 'completion_expected_at', 'error_message', 'progress', 'webhook_uri', 'webhook_method', 'scheduled_at', 'started_at', 'ended_at', 'allow_cleanup'];
 
 	/**
 	 * @var string[]
 	 */
-	public static array $fields = ['id', 'lastUpdated', 'type', 'input', 'output', 'status', 'userId', 'appId', 'customId', 'completionExpectedAt', 'errorMessage', 'progress', 'webhookUri', 'webhookMethod', 'scheduledAt', 'startedAt', 'endedAt'];
+	public static array $fields = ['id', 'lastUpdated', 'type', 'input', 'output', 'status', 'userId', 'appId', 'customId', 'completionExpectedAt', 'errorMessage', 'progress', 'webhookUri', 'webhookMethod', 'scheduledAt', 'startedAt', 'endedAt', 'allowCleanup'];
 
 
 	public function __construct() {
@@ -94,6 +97,7 @@ public function __construct() {
 		$this->addType('scheduledAt', 'integer');
 		$this->addType('startedAt', 'integer');
 		$this->addType('endedAt', 'integer');
+		$this->addType('allowCleanup', 'integer');
 	}
 
 	public function toRow(): array {
@@ -122,6 +126,7 @@ public static function fromPublicTask(OCPTask $task): self {
 			'scheduledAt' => $task->getScheduledAt(),
 			'startedAt' => $task->getStartedAt(),
 			'endedAt' => $task->getEndedAt(),
+			'allowCleanup' => $task->getAllowCleanup() ? 1 : 0,
 		]);
 		return $taskEntity;
 	}
@@ -144,6 +149,7 @@ public function toPublicTask(): OCPTask {
 		$task->setScheduledAt($this->getScheduledAt());
 		$task->setStartedAt($this->getStartedAt());
 		$task->setEndedAt($this->getEndedAt());
+		$task->setAllowCleanup($this->getAllowCleanup() !== 0);
 		return $task;
 	}
 }