feat(audio-chat): record audio, store file, submit new fake message

Signed-off-by: Julien Veyssier <[email protected]>

lib/TaskProcessing/AudioToAudioChatProvider.php

@@ -38,7 +38,7 @@ public function getId(): string {
 	}
 
 	public function getName(): string {
-		return $this->l->t('Generic Assistant');
+		return $this->l->t('Assistant fallback');
 	}
 
 	public function getTaskTypeId(): string {

src/components/ChattyLLM/ChattyLLMInputForm.vue

@@ -128,7 +128,8 @@
 				v-model:chat-content="chatContent"
 				class="session-area__input-area"
 				:loading="loading"
-				@submit="handleSubmit" />
+				@submit="handleSubmit"
+				@submit-audio="handleSubmitAudio" />
 		</NcAppContent>
 		<NcDialog :open="sessionIdToDelete !== null"
 			:name="t('assistant', 'Conversation deletion')"
@@ -415,10 +416,32 @@ export default {
 				return
 			}
 
-			console.debug('submit:', event)
 			const role = Roles.HUMAN
 			const content = this.chatContent.trim()
 			const timestamp = +new Date() / 1000 | 0
+			console.debug('[Assistant] submit text', content)
+
+			if (this.active === null) {
+				await this.newSession()
+			}
+
+			// sending a message if there are pending actions means the user rejected the actions
+			// so we can consider the agency confirmation answered
+			if (this.active.sessionAgencyPendingActions) {
+				this.active.agencyAnswered = true
+			}
+
+			this.messages.push({ role, content, timestamp })
+			this.chatContent = ''
+			this.scrollToBottom()
+			await this.newMessage(role, content, timestamp, this.active.id)
+		},
+
+		async handleSubmitAudio(fileId) {
+			console.debug('[Assistant] submit audio', fileId)
+			const role = Roles.HUMAN
+			const content = 'lala' + fileId
+			const timestamp = +new Date() / 1000 | 0
 
 			if (this.active === null) {
 				await this.newSession()
@@ -679,7 +702,7 @@ export default {
 		async pollGenerationTask(taskId, sessionId) {
 			return new Promise((resolve, reject) => {
 				this.pollMessageGenerationTimerId = setInterval(() => {
-					if (sessionId !== this.active.id) {
+					if (this.active === null || sessionId !== this.active.id) {
 						console.debug('Stop polling messages for session ' + sessionId + ' because it is not selected anymore')
 						clearInterval(this.pollMessageGenerationTimerId)
 						return
@@ -715,7 +738,7 @@ export default {
 		async pollTitleGenerationTask(taskId, sessionId) {
 			return new Promise((resolve, reject) => {
 				this.pollTitleGenerationTimerId = setInterval(() => {
-					if (sessionId !== this.active.id) {
+					if (this.active === null || sessionId !== this.active.id) {
 						console.debug('Stop polling title for session ' + sessionId + ' because it is not selected anymore')
 						clearInterval(this.pollTitleGenerationTimerId)
 						return

src/components/ChattyLLM/InputArea.vue

@@ -18,7 +18,8 @@
 			@update:model-value="$emit('update:chatContent', $event)"
 			@submit="$emit('submit', $event)" />
 		<div class="input-area__button-box">
-			<NcButton class="input-area__button-box__button"
+			<NcButton v-if="chatContent"
+				class="input-area__button-box__button"
 				:aria-label="submitBtnAriaText"
 				:disabled="disabled || !chatContent.trim()"
 				variant="primary"
@@ -27,18 +28,28 @@
 					<SendIcon :size="20" />
 				</template>
 			</NcButton>
+			<AudioRecorderWrapper v-else
+				v-model:is-recording="isRecording"
+				:compact="true"
+				:disabled="disabled"
+				@new-recording="onNewRecording" />
 		</div>
 	</div>
 </template>
 
 <script>
 import SendIcon from 'vue-material-design-icons/Send.vue'
 
-import isMobile from '../../mixins/isMobile.js'
-
 import NcButton from '@nextcloud/vue/components/NcButton'
 import NcRichContenteditable from '@nextcloud/vue/components/NcRichContenteditable'
 
+import AudioRecorderWrapper from '../fields/AudioRecorderWrapper.vue'
+
+import isMobile from '../../mixins/isMobile.js'
+import { generateOcsUrl } from '@nextcloud/router'
+import axios from '@nextcloud/axios'
+import { showError } from '@nextcloud/dialogs'
+
 /*
 maxlength calculation (just a rough estimate):
 - 1600 characters
@@ -50,8 +61,8 @@ export default {
 	name: 'InputArea',
 
 	components: {
+		AudioRecorderWrapper,
 		SendIcon,
-
 		NcButton,
 		NcRichContenteditable,
 	},
@@ -80,13 +91,18 @@ export default {
 		},
 	},
 
-	emits: ['update:chatContent', 'submit'],
+	emits: [
+		'update:chatContent',
+		'submit',
+		'submit-audio',
+	],
 
 	data: () => {
 		return {
 			placeholderText: t('assistant', 'Type a message…'),
 			thinkingText: t('assistant', 'Processing…'),
 			submitBtnAriaText: t('assistant', 'Submit'),
+			isRecording: false,
 		}
 	},
 
@@ -106,6 +122,18 @@ export default {
 				this.$refs.richContenteditable.focus()
 			})
 		},
+		onNewRecording(blob) {
+			const url = generateOcsUrl('/apps/assistant/api/v1/input-file')
+			const formData = new FormData()
+			formData.append('data', blob)
+			formData.append('filename', 'chat-input.mp3')
+			axios.post(url, formData).then(response => {
+				this.$emit('submit-audio', response.data.ocs.data.fileId)
+			}).catch(error => {
+				showError(t('assistant', 'Could not upload the recorded file'))
+				console.error(error)
+			})
+		},
 	},
 }
 </script>

src/components/fields/AudioRecorderWrapper.vue

@@ -5,13 +5,15 @@
 <template>
 	<div class="assistant-audio-recorder-wrapper">
 		<NcButton v-if="!isRecording"
-			ref="startRecordingButton"
 			:disabled="disabled"
+			:title="compact ? t('assistant', 'Start recording') : undefined"
 			@click="start">
 			<template #icon>
 				<MicrophoneOutlineIcon />
 			</template>
-			{{ t('assistant', 'Start recording') }}
+			<template v-if="!compact" #default>
+				{{ t('assistant', 'Start recording') }}
+			</template>
 		</NcButton>
 		<NcButton v-if="isRecording"
 			variant="error"
@@ -73,6 +75,10 @@ export default {
 			type: Boolean,
 			default: false,
 		},
+		compact: {
+			type: Boolean,
+			default: false,
+		},
 	},
 
 	emits: [
@@ -127,8 +133,30 @@ export default {
 				await register(await connect())
 				OCA.Assistant.encoderRegistered = true
 			}
-			const stream = await navigator.mediaDevices.getUserMedia({ audio: true })
-			this.mediaRecorder = new MediaRecorder(stream, { mimeType: 'audio/wav' })
+			// Create new audio stream
+			try {
+				this.audioStream = await navigator.mediaDevices.getUserMedia({ audio: true, video: false })
+			} catch (exception) {
+				console.debug(exception)
+				this.killStreams()
+				if (exception.name === 'NotAllowedError') {
+					showError(t('assistant', 'Access to the microphone was denied'))
+				} else {
+					showError(t('assistant', 'Microphone either not available or disabled in settings'))
+				}
+				return
+			}
+
+			// Create a media recorder to capture the stream
+			try {
+				this.mediaRecorder = new MediaRecorder(this.audioStream, { mimeType: 'audio/wav' })
+			} catch (exception) {
+				console.debug(exception)
+				this.killStreams()
+				this.audioStream = null
+				showError(t('spreed', 'Error while recording audio'))
+				return
+			}
 
 			// Add event handler to onstop
 			this.mediaRecorder.onstop = this.generateFile