Skip to content

Add support for transcribe image and audio transcription for gemini, anthropic, mistral and ollama. #1828

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
rajeevrajeshuni wants to merge 7 commits into from
Closed

Add support for transcribe image and audio transcription for gemini, anthropic, mistral and ollama. #1828

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
02b1778
Adding support for audio/image transcription for all other providers
rajeevrajeshuni Nov 25, 2025
09fbf22
uv lock version revert
rajeevrajeshuni Nov 25, 2025
57195fb
Merge branch 'dev' into issue-1767
rajeevrajeshuni Nov 28, 2025
8e5f14d
resolving merge conflicts
rajeevrajeshuni Dec 10, 2025
d57d188
resolving merge conflicts
rajeevrajeshuni Dec 10, 2025
6260f9e
strandardizing return type for transcription and some CR changes
rajeevrajeshuni Dec 11, 2025
13c034e
Merge branch 'dev' into issue-1767
dexters1 Dec 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 0 additions & 13 deletions cognee/infrastructure/llm/LLMGateway.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,19 +37,6 @@ def acreate_structured_output(
**kwargs,
)

@staticmethod
def create_structured_output(
text_input: str, system_prompt: str, response_model: Type[BaseModel]
) -> BaseModel:
from cognee.infrastructure.llm.structured_output_framework.litellm_instructor.llm.get_llm_client import (
get_llm_client,
)

llm_client = get_llm_client()
return llm_client.create_structured_output(
text_input=text_input, system_prompt=system_prompt, response_model=response_model
)

@staticmethod
def create_transcript(input) -> Coroutine:
from cognee.infrastructure.llm.structured_output_framework.litellm_instructor.llm.get_llm_client import (
Expand Down
29 changes: 17 additions & 12 deletions ...nfrastructure/llm/structured_output_framework/litellm_instructor/llm/anthropic/adapter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@
from pydantic import BaseModel
import litellm
import instructor
import anthropic
from cognee.shared.logging_utils import get_logger
from cognee.modules.observability.get_observe import get_observe
from tenacity import (
retry,
stop_after_delay,
Expand All @@ -12,38 +14,41 @@
before_sleep_log,
)

from cognee.infrastructure.llm.structured_output_framework.litellm_instructor.llm.llm_interface import (
LLMInterface,
from cognee.infrastructure.llm.structured_output_framework.litellm_instructor.llm.generic_llm_api.adapter import (
GenericAPIAdapter,
)
from cognee.shared.rate_limiting import llm_rate_limiter_context_manager
from cognee.infrastructure.llm.config import get_llm_config

logger = get_logger()
observe = get_observe()


class AnthropicAdapter(LLMInterface):
class AnthropicAdapter(GenericAPIAdapter):
"""
Adapter for interfacing with the Anthropic API, enabling structured output generation
and prompt display.
"""
Comment on lines +27 to 31
Copy link
Contributor

@coderabbitai coderabbitai bot Dec 16, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major

🧩 Analysis chain

🏁 Script executed:

#!/bin/bash
# Find and examine the GenericAPIAdapter class
find . -type f -name "*.py" | xargs grep -l "class GenericAPIAdapter"

Repository: topoteretes/cognee

Length of output: 167

🏁 Script executed:

#!/bin/bash
# Find adapter files in the repository
find . -path "*/structured_output_framework/*" -name "*.py" -type f | head -20

Repository: topoteretes/cognee

Length of output: 1814

🏁 Script executed:

#!/bin/bash
# Search for the file mentioned in the review
find . -path "**/litellm_instructor/llm/anthropic/**" -name "*.py" | head -10
# Also search for GenericAPIAdapter
find . -type f -name "*.py" | xargs grep -l "class GenericAPIAdapter" 2>/dev/null

Repository: topoteretes/cognee

Length of output: 368

🏁 Script executed:

#!/bin/bash
# Look for the adapter and parent class files
find . -type f -name "adapter.py" | grep -E "(generic|anthropic)" | head -10
rg "class GenericAPIAdapter" --type=py
rg "def create_transcript|def transcribe_image" --type=py -A 5

Repository: topoteretes/cognee

Length of output: 9839

🏁 Script executed:

#!/bin/bash
# Locate the AnthropicAdapter file and GenericAPIAdapter 
find . -name "adapter.py" -path "*anthropic*" -o -name "*.py" -path "*generic*api*"
# Search for the class definition
rg "class GenericAPIAdapter"
# Search for the methods mentioned
rg "(create_transcript|transcribe_image)" -A 3

Repository: topoteretes/cognee

Length of output: 12569

🏁 Script executed:

#!/bin/bash
# First, let's explore the directory structure
find cognee -path "*/litellm_instructor/llm/anthropic/*" -type f 2>/dev/null || echo "Path not found"
find cognee -name "*adapter*" -type f 2>/dev/null | head -10

Repository: topoteretes/cognee

Length of output: 859

🏁 Script executed:

#!/bin/bash
# Check if the file exists and examine it
ls -la cognee/infrastructure/llm/structured_output_framework/litellm_instructor/llm/anthropic/adapter.py 2>/dev/null || \
find cognee -name "adapter.py" | head -5

# Search for GenericAPIAdapter
rg "class GenericAPIAdapter" -A 10
# Search for transcription methods
rg "def (create_transcript|transcribe_image)" -B 2 -A 10

Repository: topoteretes/cognee

Length of output: 19540

🏁 Script executed:

#!/bin/bash
#!/bin/bash
# Search for GenericAPIAdapter and related methods
rg "GenericAPIAdapter" cognee --type=py
rg "create_transcript|transcribe_image" cognee --type=py -B 1 -A 3
# Find adapter files
fd "adapter.py" cognee

Repository: topoteretes/cognee

Length of output: 18772

🏁 Script executed:

#!/bin/bash
find . -name "*.py" -type f -path "*litellm_instructor*" | head -20
find . -name "*.py" -type f | xargs grep -l "class GenericAPIAdapter" | head -5
find . -name "*.py" -type f | xargs grep -l "create_transcript\|transcribe_image" | head -10

Repository: topoteretes/cognee

Length of output: 2918

🏁 Script executed:

#!/bin/bash
# List files in the anthropic directory
if [ -d "cognee/infrastructure/llm/structured_output_framework/litellm_instructor/llm/anthropic" ]; then
  ls -la cognee/infrastructure/llm/structured_output_framework/litellm_instructor/llm/anthropic/
fi

# Find all adapter.py files in the structured_output_framework
find cognee/infrastructure/llm/structured_output_framework -name "adapter.py" -o -name "*adapter*"

# Search for GenericAPIAdapter
grep -r "class GenericAPIAdapter" . --include="*.py" 2>/dev/null | head -5

Repository: topoteretes/cognee

Length of output: 1117

🏁 Script executed:

#!/bin/bash
# Get directory structure of the framework
find cognee/infrastructure -type d -name "*litellm*" -o -type d -name "*structured*" | head -10
# List the files in the anthropic directory if it exists
find cognee -path "*anthropic*" -name "*.py" 2>/dev/null | head -10
# Find all adapter files
find cognee -name "*.py" -type f | xargs grep -l "class.*Adapter"

Repository: topoteretes/cognee

Length of output: 2373

🏁 Script executed:

#!/bin/bash
# Search for GenericAPIAdapter definition
rg "class GenericAPIAdapter" --type py -C 5
# Search for create_transcript method definition
rg "def create_transcript" --type py -C 3
# Search for transcribe_image method definition  
rg "def transcribe_image" --type py -C 3
# List all adapter.py files
find . -name "adapter.py" -type f

Repository: topoteretes/cognee

Length of output: 12294

🏁 Script executed:

#!/bin/bash
# Find the specific files mentioned in the review
find . -type f -name "*.py" | xargs grep -l "class.*Adapter" | head -20

# Find files in the anthropic directory
find . -type f -path "*anthropic*" -name "*.py" | head -20

# Look for create_transcript and transcribe_image definitions
grep -r "def create_transcript" . --include="*.py" | head -10
grep -r "def transcribe_image" . --include="*.py" | head -10

# Look for GenericAPIAdapter
grep -r "class GenericAPIAdapter" . --include="*.py"

Repository: topoteretes/cognee

Length of output: 3498

🏁 Script executed:

#!/bin/bash
# Step 1: List all files in the structured_output_framework directory
find . -path "*structured_output_framework*" -name "*.py" -type f 2>/dev/null | head -30

# Step 2: Find all adapter files
find . -name "*adapter*" -type f 2>/dev/null | grep -E "\.py$"

# Step 3: Search for GenericAPIAdapter and related methods
rg "(class GenericAPIAdapter|def create_transcript|def transcribe_image)" --type=py -B 2 -A 5

Repository: topoteretes/cognee

Length of output: 17733

🏁 Script executed:

#!/bin/bash
set -e

# Find all Python files in cognee/infrastructure/llm
find cognee/infrastructure/llm -name "*.py" -type f 2>/dev/null > /tmp/files.txt

# Count them
wc -l /tmp/files.txt

# Show first 30 files
head -30 /tmp/files.txt

# Search for GenericAPIAdapter in any Python file
echo "=== Searching for GenericAPIAdapter ==="
grep -r "class GenericAPIAdapter" . --include="*.py" 2>/dev/null || echo "Not found in grep"

# Search for the AnthropicAdapter file
echo "=== Looking for AnthropicAdapter ==="
find . -path "*anthropic*adapter*" -name "*.py" 2>/dev/null

# Search for create_transcript method
echo "=== Looking for create_transcript method ==="
grep -r "def create_transcript" . --include="*.py" 2>/dev/null | head -5

# Search for transcribe_image method
echo "=== Looking for transcribe_image method ==="
grep -r "def transcribe_image" . --include="*.py" 2>/dev/null | head -5

Repository: topoteretes/cognee

Length of output: 4132

🏁 Script executed:

#!/bin/bash
# Comprehensive search to understand the adapter architecture
echo "Searching for adapter files..."
find . -name "*.py" -type f | xargs grep -l "class.*Adapter" 2>/dev/null | grep -E "(generic|anthropic|adapter)" | head -10

echo -e "\n Looking for GenericAPIAdapter..."
find . -name "*.py" -type f | xargs grep "class GenericAPIAdapter" 2>/dev/null

echo -e "\nLooking for create_transcript and transcribe_image..."
find . -name "*.py" -type f | xargs grep -E "(def create_transcript|def transcribe_image)" 2>/dev/null

Repository: topoteretes/cognee

Length of output: 2734

🏁 Script executed:

#!/bin/bash
# Find all .py files that might contain GenericAPIAdapter
rg "GenericAPIAdapter" --type py
# Find create_transcript and transcribe_image  
rg "create_transcript|transcribe_image" --type py -B 2 -A 5
# List structured output framework files
fd . cognee/infrastructure/llm/structured_output_framework --type f --extension py | head -20

Repository: topoteretes/cognee

Length of output: 25949

🏁 Script executed:

#!/bin/bash
# Search for GenericAPIAdapter definition
grep -r "class GenericAPIAdapter" cognee --include="*.py"

# Search for AnthropicAdapter
grep -r "class AnthropicAdapter" cognee --include="*.py"

# Search for methods
grep -r "def create_transcript\|def transcribe_image" cognee --include="*.py" -A 3

# Check directory structure
find cognee/infrastructure/llm -type d | head -15

Repository: topoteretes/cognee

Length of output: 7159

🏁 Script executed:

#!/bin/bash
# Direct search for the adapter files and their methods
find cognee -name "adapter.py" 2>/dev/null -exec grep -H "class\|def create_transcript\|def transcribe_image" {} \;

# Search for GenericAPIAdapter definition
rg "class GenericAPIAdapter" --type py --no-heading -A 15 2>/dev/null | head -30

# Search for the methods
rg "def (create_transcript|transcribe_image)" --type py --no-heading -B 1 -A 10

Repository: topoteretes/cognee

Length of output: 22727

🏁 Script executed:

#!/bin/bash
# Final comprehensive search
echo "=== Searching for GenericAPIAdapter ==="
rg "class GenericAPIAdapter" --type py
echo "=== Searching for transcription methods ==="
rg "def (create_transcript|transcribe_image)" --type py  
echo "=== Listing adapter files ==="
find cognee -name "*adapter*" -type f -name "*.py"
echo "=== Checking litellm_instructor structure ==="
ls -la cognee/infrastructure/llm/structured_output_framework/litellm_instructor/llm/ 2>/dev/null || echo "Directory not accessible"

Repository: topoteretes/cognee

Length of output: 5023

🏁 Script executed:

#!/bin/bash
# Check if the adapter classes and methods actually exist
git ls-files | grep -E "(adapter|instructor)" | head -20
cat cognee/infrastructure/llm/structured_output_framework/litellm_instructor/llm/anthropic/adapter.py 2>/dev/null || echo "File not found"
grep -r "GenericAPIAdapter" cognee --include="*.py" -l
grep -r "create_transcript\|transcribe_image" cognee --include="*.py" -B 1 -A 3

Repository: topoteretes/cognee

Length of output: 19507

🏁 Script executed:

#!/bin/bash
# Simple direct existence and content check
if [ -f "cognee/infrastructure/llm/structured_output_framework/litellm_instructor/llm/anthropic/adapter.py" ]; then
    echo "File exists, content:"
    cat cognee/infrastructure/llm/structured_output_framework/litellm_instructor/llm/anthropic/adapter.py
else
    echo "File not found at expected path"
fi

# Find adapter.py files
find cognee -name "adapter.py"

# Check for GenericAPIAdapter class
rg "class GenericAPIAdapter" cognee/

# Check for the methods in question
rg "def create_transcript" cognee/ -A 10
rg "def transcribe_image" cognee/ -A 10

Repository: topoteretes/cognee

Length of output: 20137

Override create_transcript with explicit error handling for Anthropic audio limitation.

AnthropicAdapter inherits create_transcript from GenericAPIAdapter, but Anthropic's API does not support audio transcription. Override this method to raise NotImplementedError with a clear message directing users to external transcription services. For transcribe_image, verify the inherited implementation works correctly with Anthropic's vision API by testing with base64-encoded images.


name = "Anthropic"
model: str
default_instructor_mode = "anthropic_tools"

def __init__(self, max_completion_tokens: int, model: str = None, instructor_mode: str = None):
import anthropic

def __init__(
self, api_key: str, model: str, max_completion_tokens: int, instructor_mode: str = None
):
super().__init__(
api_key=api_key,
model=model,
max_completion_tokens=max_completion_tokens,
name="Anthropic",
)
self.instructor_mode = instructor_mode if instructor_mode else self.default_instructor_mode

self.aclient = instructor.patch(
create=anthropic.AsyncAnthropic(api_key=get_llm_config().llm_api_key).messages.create,
create=anthropic.AsyncAnthropic(api_key=self.api_key).messages.create,
mode=instructor.Mode(self.instructor_mode),
)

self.model = model
self.max_completion_tokens = max_completion_tokens

@observe(as_type="generation")
@retry(
stop=stop_after_delay(128),
wait=wait_exponential_jitter(8, 128),
Expand Down
56 changes: 31 additions & 25 deletions ...e/infrastructure/llm/structured_output_framework/litellm_instructor/llm/gemini/adapter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
"""Adapter for Generic API LLM provider API"""
"""Adapter for Gemini API LLM provider"""

import litellm
import instructor
Expand All @@ -8,13 +8,9 @@
from litellm.exceptions import ContentPolicyViolationError
from instructor.core import InstructorRetryException

from cognee.infrastructure.llm.exceptions import ContentPolicyFilterError
from cognee.infrastructure.llm.structured_output_framework.litellm_instructor.llm.llm_interface import (
LLMInterface,
)
import logging
from cognee.shared.rate_limiting import llm_rate_limiter_context_manager
from cognee.shared.logging_utils import get_logger

from tenacity import (
retry,
stop_after_delay,
Expand All @@ -23,55 +19,65 @@
before_sleep_log,
)

from cognee.infrastructure.llm.exceptions import ContentPolicyFilterError
from cognee.infrastructure.llm.structured_output_framework.litellm_instructor.llm.generic_llm_api.adapter import (
GenericAPIAdapter,
)
from cognee.shared.logging_utils import get_logger
from cognee.modules.observability.get_observe import get_observe

logger = get_logger()
observe = get_observe()


class GeminiAdapter(LLMInterface):
class GeminiAdapter(GenericAPIAdapter):
"""
Adapter for Gemini API LLM provider.
This class initializes the API adapter with necessary credentials and configurations for
interacting with the gemini LLM models. It provides methods for creating structured outputs
based on user input and system prompts.
based on user input and system prompts, as well as multimodal processing capabilities.
Public methods:
- acreate_structured_output(text_input: str, system_prompt: str, response_model:
Type[BaseModel]) -> BaseModel
- acreate_structured_output(text_input: str, system_prompt: str, response_model: Type[BaseModel]) -> BaseModel
- create_transcript(input) -> BaseModel: Transcribe audio files to text
- transcribe_image(input) -> BaseModel: Inherited from GenericAPIAdapter
"""

name: str
model: str
api_key: str
default_instructor_mode = "json_mode"

def __init__(
self,
endpoint,
api_key: str,
model: str,
api_version: str,
max_completion_tokens: int,
endpoint: str = None,
api_version: str = None,
transcription_model: str = None,
instructor_mode: str = None,
fallback_model: str = None,
fallback_api_key: str = None,
fallback_endpoint: str = None,
):
self.model = model
self.api_key = api_key
self.endpoint = endpoint
self.api_version = api_version
self.max_completion_tokens = max_completion_tokens

self.fallback_model = fallback_model
self.fallback_api_key = fallback_api_key
self.fallback_endpoint = fallback_endpoint

super().__init__(
api_key=api_key,
model=model,
max_completion_tokens=max_completion_tokens,
name="Gemini",
endpoint=endpoint,
api_version=api_version,
transcription_model=transcription_model,
fallback_model=fallback_model,
fallback_api_key=fallback_api_key,
fallback_endpoint=fallback_endpoint,
)
Comment on lines 49 to +73
Copy link
Contributor

@coderabbitai coderabbitai bot Dec 16, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🛠️ Refactor suggestion | 🟠 Major

Add missing image_transcribe_model parameter and docstring.

Two issues:

  1. Missing parameter: GenericAPIAdapter.__init__ accepts an image_transcribe_model parameter (see line 57 in the base class), but GeminiAdapter.__init__ does not accept or pass this parameter. This means image transcription will default to using the main model instead of a potentially specialized image model.

  2. Missing docstring: Per the coding guidelines, undocumented function definitions are assumed incomplete. Please add a docstring documenting the parameters and their purposes.

Apply this diff to add the missing parameter:

     def __init__(
         self,
         api_key: str,
         model: str,
         max_completion_tokens: int,
         endpoint: str = None,
         api_version: str = None,
         transcription_model: str = None,
+        image_transcribe_model: str = None,
         instructor_mode: str = None,
         fallback_model: str = None,
         fallback_api_key: str = None,
         fallback_endpoint: str = None,
     ):
+        """
+        Initialize the Gemini adapter with API credentials and model configurations.
+        
+        Parameters:
+        -----------
+            api_key: Gemini API key
+            model: Primary model identifier for structured output generation
+            max_completion_tokens: Maximum tokens for completion responses
+            endpoint: Optional custom API endpoint
+            api_version: Optional API version
+            transcription_model: Optional model for audio transcription (defaults to main model)
+            image_transcribe_model: Optional model for image transcription (defaults to main model)
+            instructor_mode: Mode for instructor client (defaults to "json_mode")
+            fallback_model: Fallback model for content policy violations
+            fallback_api_key: API key for fallback model
+            fallback_endpoint: Endpoint for fallback model
+        """
         super().__init__(
             api_key=api_key,
             model=model,
             max_completion_tokens=max_completion_tokens,
             name="Gemini",
             endpoint=endpoint,
             api_version=api_version,
             transcription_model=transcription_model,
+            image_transcribe_model=image_transcribe_model,
             fallback_model=fallback_model,
             fallback_api_key=fallback_api_key,
             fallback_endpoint=fallback_endpoint,
         )

As per coding guidelines, undocumented function definitions are considered incomplete.

self.instructor_mode = instructor_mode if instructor_mode else self.default_instructor_mode

self.aclient = instructor.from_litellm(
litellm.acompletion, mode=instructor.Mode(self.instructor_mode)
)

@observe(as_type="generation")
@retry(
stop=stop_after_delay(128),
wait=wait_exponential_jitter(8, 128),
Expand Down
139 changes: 132 additions & 7 deletions ...ructure/llm/structured_output_framework/litellm_instructor/llm/generic_llm_api/adapter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
"""Adapter for Generic API LLM provider API"""

import base64
import mimetypes
import litellm
import instructor
from typing import Type
from typing import Type, Optional
from pydantic import BaseModel
from openai import ContentFilterFinishReasonError
from litellm.exceptions import ContentPolicyViolationError
Expand All @@ -12,6 +14,8 @@
from cognee.infrastructure.llm.structured_output_framework.litellm_instructor.llm.llm_interface import (
LLMInterface,
)
from cognee.infrastructure.files.utils.open_data_file import open_data_file
from cognee.modules.observability.get_observe import get_observe
import logging
from cognee.shared.rate_limiting import llm_rate_limiter_context_manager
from cognee.shared.logging_utils import get_logger
Expand All @@ -23,7 +27,10 @@
before_sleep_log,
)

from ..types import TranscriptionReturnType

logger = get_logger()
observe = get_observe()


class GenericAPIAdapter(LLMInterface):
Expand All @@ -39,18 +46,19 @@ class GenericAPIAdapter(LLMInterface):
Type[BaseModel]) -> BaseModel
"""

name: str
model: str
api_key: str
MAX_RETRIES = 5
default_instructor_mode = "json_mode"

def __init__(
self,
endpoint,
api_key: str,
model: str,
name: str,
max_completion_tokens: int,
name: str,
endpoint: str = None,
api_version: str = None,
transcription_model: str = None,
image_transcribe_model: str = None,
instructor_mode: str = None,
fallback_model: str = None,
fallback_api_key: str = None,
Expand All @@ -59,9 +67,11 @@ def __init__(
self.name = name
self.model = model
self.api_key = api_key
self.api_version = api_version
self.endpoint = endpoint
self.max_completion_tokens = max_completion_tokens

self.transcription_model = transcription_model or model
self.image_transcribe_model = image_transcribe_model or model
self.fallback_model = fallback_model
self.fallback_api_key = fallback_api_key
self.fallback_endpoint = fallback_endpoint
Expand All @@ -72,6 +82,7 @@ def __init__(
litellm.acompletion, mode=instructor.Mode(self.instructor_mode)
)

@observe(as_type="generation")
@retry(
stop=stop_after_delay(128),
wait=wait_exponential_jitter(8, 128),
Expand Down Expand Up @@ -173,3 +184,117 @@ async def acreate_structured_output(
raise ContentPolicyFilterError(
f"The provided input contains content that is not aligned with our content policy: {text_input}"
) from error

@observe(as_type="transcription")
@retry(
stop=stop_after_delay(128),
wait=wait_exponential_jitter(2, 128),
retry=retry_if_not_exception_type(litellm.exceptions.NotFoundError),
before_sleep=before_sleep_log(logger, logging.DEBUG),
reraise=True,
)
async def create_transcript(self, input) -> Optional[TranscriptionReturnType]:
"""
Generate an audio transcript from a user query.

This method creates a transcript from the specified audio file, raising a
FileNotFoundError if the file does not exist. The audio file is processed and the
transcription is retrieved from the API.

Parameters:
-----------
- input: The path to the audio file that needs to be transcribed.

Returns:
--------
The generated transcription of the audio file.
"""
async with open_data_file(input, mode="rb") as audio_file:
encoded_string = base64.b64encode(audio_file.read()).decode("utf-8")
mime_type, _ = mimetypes.guess_type(input)
if not mime_type or not mime_type.startswith("audio/"):
raise ValueError(
f"Could not determine MIME type for audio file: {input}. Is the extension correct?"
)
response = litellm.completion(
model=self.transcription_model,
messages=[
{
"role": "user",
"content": [
{
"type": "file",
"file": {"file_data": f"data:{mime_type};base64,{encoded_string}"},
},
{"type": "text", "text": "Transcribe the following audio precisely."},
],
}
],
api_key=self.api_key,
api_version=self.api_version,
max_completion_tokens=self.max_completion_tokens,
api_base=self.endpoint,
max_retries=self.MAX_RETRIES,
)
if response and response.choices and len(response.choices) > 0:
return TranscriptionReturnType(response.choices[0].message.content,response)
else:
return None


@observe(as_type="transcribe_image")
@retry(
stop=stop_after_delay(128),
wait=wait_exponential_jitter(2, 128),
retry=retry_if_not_exception_type(litellm.exceptions.NotFoundError),
before_sleep=before_sleep_log(logger, logging.DEBUG),
reraise=True,
)
async def transcribe_image(self, input) -> Optional[BaseModel]:
"""
Generate a transcription of an image from a user query.

This method encodes the image and sends a request to the API to obtain a
description of the contents of the image.

Parameters:
-----------
- input: The path to the image file that needs to be transcribed.

Returns:
--------
- BaseModel: A structured output generated by the model, returned as an instance of
BaseModel.
"""
async with open_data_file(input, mode="rb") as image_file:
encoded_image = base64.b64encode(image_file.read()).decode("utf-8")
mime_type, _ = mimetypes.guess_type(input)
if not mime_type or not mime_type.startswith("image/"):
raise ValueError(
f"Could not determine MIME type for image file: {input}. Is the extension correct?"
)
return litellm.completion(
model=self.image_transcribe_model,
messages=[
{
"role": "user",
"content": [
{
"type": "text",
"text": "What's in this image?",
},
{
"type": "image_url",
"image_url": {
"url": f"data:{mime_type};base64,{encoded_image}",
},
},
],
}
],
api_key=self.api_key,
api_base=self.endpoint,
api_version=self.api_version,
max_completion_tokens=300,
max_retries=self.MAX_RETRIES,
)
Loading
Loading