Skip to content

Implement audio chat provider #227

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
julien-nc merged 11 commits into from
Jul 10, 2025
Merged

Implement audio chat provider #227

Changes from 1 commit
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
f178063
implement audio chat provider
julien-nc Jul 1, 2025
6334cae
adjust AudioToAudioChatTaskType
julien-nc Jul 3, 2025
e3a6d2d
feat(audio-chat): add condition to register provider and task type
julien-nc Jul 3, 2025
b875987
review adjustments
julien-nc Jul 3, 2025
2054ea6
fix tests
julien-nc Jul 3, 2025
456a8a1
change defaults and enum values if using openai or not, use llmModel …
julien-nc Jul 4, 2025
f00717e
use service name in logs instead of hardcoded value, fall back to app ID
julien-nc Jul 4, 2025
47b9c86
remove fallback audio chat task type, register the provider only if t…
julien-nc Jul 7, 2025
42c644c
isolate oneStep audio2audio processing in a provider method
julien-nc Jul 9, 2025
cd67b7f
try to return the remote audio ID when using the chat endpoint with a…
julien-nc Jul 9, 2025
3ea9b77
return remote audio expiration date
julien-nc Jul 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
use service name in logs instead of hardcoded value, fall back to app ID 
Signed-off-by: Julien Veyssier <[email protected]>
  • Loading branch information
@julien-nc
julien-nc committed Jul 10, 2025
commit f00717ea9a39daef286028f802866fe0fe2164a7
20 changes: 11 additions & 9 deletions lib/TaskProcessing/AudioToAudioChatProvider.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,8 @@ public function process(?string $userId, array $input, callable $reportProgress)

$sttModel = $this->appConfig->getValueString(Application::APP_ID, 'default_stt_model_id', Application::DEFAULT_MODEL_ID) ?: Application::DEFAULT_MODEL_ID;

$serviceName = $this->appConfig->getValueString(Application::APP_ID, 'service_name') ?: Application::APP_ID;

/////////////// Using the chat API if connected to OpenAI
if ($this->openAiAPIService->isUsingOpenAi()) {
$b64Audio = base64_encode($inputFile->getContent());
Expand All @@ -216,7 +218,7 @@ public function process(?string $userId, array $input, callable $reportProgress)
$inputTranscription = $this->openAiAPIService->transcribeFile($userId, $inputFile, false, $sttModel);
$result['input_transcript'] = $inputTranscription;
} catch (Exception $e) {
$this->logger->warning('OpenAI\'s Whisper transcription failed with: ' . $e->getMessage(), ['exception' => $e]);
$this->logger->warning($serviceName . ' transcription failed with: ' . $e->getMessage(), ['exception' => $e]);
}

return $result;
Expand All @@ -227,19 +229,19 @@ public function process(?string $userId, array $input, callable $reportProgress)
try {
$inputTranscription = $this->openAiAPIService->transcribeFile($userId, $inputFile, false, $sttModel);
} catch (Exception $e) {
$this->logger->warning('OpenAI\'s Whisper transcription failed with: ' . $e->getMessage(), ['exception' => $e]);
throw new RuntimeException('OpenAI\'s Whisper transcription failed with: ' . $e->getMessage());
$this->logger->warning($serviceName . ' transcription failed with: ' . $e->getMessage(), ['exception' => $e]);
throw new RuntimeException($serviceName . ' transcription failed with: ' . $e->getMessage());
}

// free prompt
try {
$completion = $this->openAiAPIService->createChatCompletion($userId, $llmModel, $inputTranscription, $systemPrompt, $history, 1, 1000);
$completion = $completion['messages'];
} catch (Exception $e) {
throw new RuntimeException('OpenAI/LocalAI request failed: ' . $e->getMessage());
throw new RuntimeException($serviceName . ' chat completion request failed: ' . $e->getMessage());
}
if (count($completion) === 0) {
throw new RuntimeException('No completion in OpenAI/LocalAI response.');
throw new RuntimeException('No completion in ' . $serviceName . ' response.');
}
$llmResult = array_pop($completion);

Expand All @@ -248,17 +250,17 @@ public function process(?string $userId, array $input, callable $reportProgress)
$apiResponse = $this->openAiAPIService->requestSpeechCreation($userId, $llmResult, $ttsModel, $outputVoice, $speed);

if (!isset($apiResponse['body'])) {
$this->logger->warning('OpenAI/LocalAI\'s text to speech generation failed: no speech returned');
throw new RuntimeException('OpenAI/LocalAI\'s text to speech generation failed: no speech returned');
$this->logger->warning($serviceName . ' text to speech generation failed: no speech returned');
throw new RuntimeException($serviceName . ' text to speech generation failed: no speech returned');
}
return [
'output' => $apiResponse['body'],
'output_transcript' => $llmResult,
'input_transcript' => $inputTranscription,
];
} catch (\Exception $e) {
$this->logger->warning('OpenAI/LocalAI\'s text to speech generation failed with: ' . $e->getMessage(), ['exception' => $e]);
throw new RuntimeException('OpenAI/LocalAI\'s text to speech generation failed with: ' . $e->getMessage());
$this->logger->warning($serviceName . ' text to speech generation failed with: ' . $e->getMessage(), ['exception' => $e]);
throw new RuntimeException($serviceName . ' text to speech generation failed with: ' . $e->getMessage());
}
}
}