feat(audio-chat): perform TTS for response after agency confirmation

Signed-off-by: Julien Veyssier <[email protected]>

lib/Controller/ChattyLLMController.php

@@ -251,7 +251,7 @@ public function getSessions(): JSONResponse {
 	 * @param string $role Role of the message (human, assistant etc...)
 	 * @param string $content Content of the message
 	 * @param int $timestamp Date of the message
-	 * @param ?list<array{type: string, fileId: int}> $attachments List of attachment objects
+	 * @param ?list<array{type: string, file_id: int}> $attachments List of attachment objects
 	 * @param bool $firstHumanMessage Is it the first human message of the session?
 	 * @return JSONResponse<Http::STATUS_OK, AssistantChatMessage, array{}>|JSONResponse<Http::STATUS_INTERNAL_SERVER_ERROR|Http::STATUS_UNAUTHORIZED|Http::STATUS_BAD_REQUEST|Http::STATUS_NOT_FOUND, array{error: string}, array{}>
 	 *
@@ -474,7 +474,7 @@ public function generateForSession(int $sessionId, int $agencyConfirm = 0): JSON
 				&& isset($this->taskProcessingManager->getAvailableTaskTypes()[\OCP\TaskProcessing\TaskTypes\ContextAgentAudioInteraction::ID])
 			) {
 				// audio agency
-				$fileId = $audioAttachment['fileId'];
+				$fileId = $audioAttachment['file_id'];
 				try {
 					$taskId = $this->scheduleAgencyAudioTask($fileId, $agencyConfirm, $lastConversationToken, $sessionId, $lastUserMessage->getId());
 				} catch (\Exception $e) {
@@ -522,7 +522,7 @@ public function generateForSession(int $sessionId, int $agencyConfirm = 0): JSON
 				&& class_exists('OCP\\TaskProcessing\\TaskTypes\\AudioToAudioChat')
 				&& isset($this->taskProcessingManager->getAvailableTaskTypes()[\OCP\TaskProcessing\TaskTypes\AudioToAudioChat::ID])
 			) {
-				$fileId = $audioAttachment['fileId'];
+				$fileId = $audioAttachment['file_id'];
 				try {
 					$taskId = $this->scheduleAudioChatTask($fileId, $systemPrompt, $history, $sessionId, $lastUserMessage->getId());
 				} catch (\Exception $e) {
@@ -589,7 +589,6 @@ public function regenerateForSession(int $sessionId, int $messageId): JSONRespon
 	 * @param int $taskId The message generation task ID
 	 * @param int $sessionId The chat session ID
 	 * @return JSONResponse<Http::STATUS_OK, AssistantChatAgencyMessage, array{}>|JSONResponse<Http::STATUS_EXPECTATION_FAILED, array{task_status: int}, array{}>|JSONResponse<Http::STATUS_INTERNAL_SERVER_ERROR|Http::STATUS_UNAUTHORIZED|Http::STATUS_BAD_REQUEST|Http::STATUS_NOT_FOUND, array{error: string}, array{}>
-	 * @throws DoesNotExistException
 	 * @throws MultipleObjectsReturnedException
 	 * @throws \OCP\DB\Exception
 	 *
@@ -631,8 +630,11 @@ public function checkMessageGenerationTask(int $taskId, int $sessionId): JSONRes
 				// do not insert here, it is done by the listener
 				return new JSONResponse($jsonMessage);
 			} catch (\OCP\DB\Exception $e) {
-				$this->logger->warning('Failed to add a chat message into DB', ['exception' => $e]);
+				$this->logger->warning('Failed to add a chat message into the DB', ['exception' => $e]);
 				return new JSONResponse(['error' => $this->l10n->t('Failed to add a chat message into DB')], Http::STATUS_INTERNAL_SERVER_ERROR);
+			} catch (DoesNotExistException $e) {
+				$this->logger->debug('Task finished successfully but failed to find the chat message in the DB. It should be created soon.', ['exception' => $e]);
+				return new JSONResponse(['task_status' => $task->getstatus()], Http::STATUS_EXPECTATION_FAILED);
 			}
 		} elseif ($task->getstatus() === Task::STATUS_RUNNING || $task->getstatus() === Task::STATUS_SCHEDULED) {
 			return new JSONResponse(['task_status' => $task->getstatus()], Http::STATUS_EXPECTATION_FAILED);

lib/Db/ChattyLLM/MessageMapper.php

@@ -62,6 +62,19 @@ public function getLastHumanMessage(int $sessionId): Message {
 		return $this->findEntity($qb);
 	}
 
+	public function getLastNonEmptyHumanMessage(int $sessionId): Message {
+		$qb = $this->db->getQueryBuilder();
+		$qb->select(Message::$columns)
+			->from($this->getTableName())
+			->where($qb->expr()->eq('session_id', $qb->createPositionalParameter($sessionId, IQueryBuilder::PARAM_INT)))
+			->andWhere($qb->expr()->eq('role', $qb->createPositionalParameter('human', IQueryBuilder::PARAM_STR)))
+			->andWhere($qb->expr()->nonEmptyString('content'))
+			->orderBy('timestamp', 'DESC')
+			->setMaxResults(1);
+
+		return $this->findEntity($qb);
+	}
+
 	/**
 	 * @param int $sessionId
 	 * @param int $cursor

lib/Listener/ChattyLLMTaskListener.php

@@ -13,9 +13,12 @@
 use OCA\Assistant\Db\ChattyLLM\Message;
 use OCA\Assistant\Db\ChattyLLM\MessageMapper;
 use OCA\Assistant\Db\ChattyLLM\SessionMapper;
+use OCA\Assistant\Service\TaskProcessingService;
 use OCP\EventDispatcher\Event;
 use OCP\EventDispatcher\IEventListener;
 use OCP\TaskProcessing\Events\TaskSuccessfulEvent;
+use OCP\TaskProcessing\Task;
+use OCP\TaskProcessing\TaskTypes\TextToSpeech;
 use Psr\Log\LoggerInterface;
 
 /**
@@ -26,6 +29,7 @@ class ChattyLLMTaskListener implements IEventListener {
 	public function __construct(
 		private MessageMapper $messageMapper,
 		private SessionMapper $sessionMapper,
+		private TaskProcessingService $taskProcessingService,
 		private LoggerInterface $logger,
 	) {
 	}
@@ -75,7 +79,7 @@ public function handle(Event $event): void {
 				$message->setContent($outputTranscript);
 				// agency might not return any output but just ask for confirmation
 				if ($outputTranscript !== '') {
-					$message->setAttachments('[{"type":"Audio","fileId":' . $task->getOutput()['output'] . '}]');
+					$message->setAttachments('[{"type":"Audio","file_id":' . $task->getOutput()['output'] . '}]');
 				}
 				// now we have the transcription of the user audio input
 				if (preg_match('/^chatty-llm:\d+:(\d+)$/', $customId, $matches)) {
@@ -85,7 +89,9 @@ public function handle(Event $event): void {
 					$this->messageMapper->update($queryMessage);
 				}
 			} else {
-				$message->setContent(trim($task->getOutput()['output'] ?? ''));
+				$content = trim($task->getOutput()['output'] ?? '');
+				$message->setContent($content);
+				$this->runTtsIfNeeded($sessionId, $message, $taskTypeId, $task->getUserId());
 			}
 			try {
 				$this->messageMapper->insert($message);
@@ -104,4 +110,48 @@ public function handle(Event $event): void {
 			}
 		}
 	}
+
+	/**
+	 * Run TTS on the response of an agency confirmation message
+	 *
+	 * @param int $sessionId
+	 * @param Message $message
+	 * @param string $taskTypeId
+	 * @param string|null $userId
+	 * @return void
+	 */
+	private function runTtsIfNeeded(int $sessionId, Message $message, string $taskTypeId, ?string $userId): void {
+		if ($taskTypeId !== \OCP\TaskProcessing\TaskTypes\ContextAgentInteraction::ID) {
+			return;
+		}
+		// is the last non-empty user message an audio one?
+		$lastNonEmptyMessage = $this->messageMapper->getLastNonEmptyHumanMessage($sessionId);
+		$attachments = $lastNonEmptyMessage->jsonSerialize()['attachments'] ?? [];
+		foreach ($attachments as $attachment) {
+			if ($attachment['type'] === 'Audio') {
+				// we found an audio attachment
+				$this->runTtsTask($message, $userId);
+				return;
+			}
+		}
+	}
+
+	/**
+	 * @param Message $message
+	 * @param string|null $userId
+	 * @return void
+	 */
+	private function runTtsTask(Message $message, ?string $userId): void {
+		$task = new Task(
+			TextToSpeech::ID,
+			['input' => $message->getContent()],
+			Application::APP_ID . ':internal',
+			$userId,
+		);
+		$ttsTaskOutput = $this->taskProcessingService->runTaskProcessingTask($task);
+		$speechFileId = $ttsTaskOutput['speech'];
+		// we need to set "ocp_task_id" here because the file is not an output of the task that produced the message
+		// and we need the task ID + the file ID to load the audio file in the frontend
+		$message->setAttachments('[{"type":"Audio","file_id":' . $speechFileId . ',"ocp_task_id":' . $task->getId() . '}]');
+	}
 }

lib/TaskProcessing/ContextAgentAudioInteractionProvider.php

@@ -33,7 +33,7 @@ public function __construct(
 	}
 
 	public function getId(): string {
-		return Application::APP_ID . '-audio2audio:chat';
+		return Application::APP_ID . '-contextagent:audio-interaction';
 	}
 
 	public function getName(): string {

openapi.json

@@ -2507,13 +2507,13 @@
                                             "type": "object",
                                             "required": [
                                                 "type",
-                                                "fileId"
+                                                "file_id"
                                             ],
                                             "properties": {
                                                 "type": {
                                                     "type": "string"
                                                 },
-                                                "fileId": {
+                                                "file_id": {
                                                     "type": "integer",
                                                     "format": "int64"
                                                 }

src/components/ChattyLLM/ChattyLLMInputForm.vue

@@ -445,7 +445,7 @@ export default {
 			const role = Roles.HUMAN
 			const content = ''
 			const timestamp = +new Date() / 1000 | 0
-			const attachments = [{ type: SHAPE_TYPE_NAMES.Audio, fileId }]
+			const attachments = [{ type: SHAPE_TYPE_NAMES.Audio, file_id: fileId }]
 
 			if (this.active === null) {
 				await this.newSession()

src/components/ChattyLLM/Message.vue

@@ -62,11 +62,11 @@
 			:references="references"
 			:autolink="true" />
 		<AudioDisplay v-for="a in audioAttachments"
-			:key="a.type + '-' + a.fileId"
+			:key="a.type + '-' + a.file_id"
 			class="message__content"
 			:autoplay="message.autoPlay"
-			:file-id="a.fileId"
-			:task-id="message.role === 'human' ? undefined : message.ocp_task_id"
+			:file-id="a.file_id"
+			:task-id="message.role === 'human' ? undefined : (a.ocp_task_id ?? message.ocp_task_id)"
 			:is-output="message.role === 'assistant'" />
 	</div>
 </template>