(azure): Enable stream_options for Azure OpenAI. (#6024)

litellm/llms/AzureOpenAI/azure.py

@@ -135,6 +135,7 @@ def get_supported_openai_params(self):
             "temperature",
             "n",
             "stream",
+            "stream_options",
             "stop",
             "max_tokens",
             "max_completion_tokens",
@@ -938,6 +939,7 @@ def streaming(
             model=model,
             custom_llm_provider="azure",
             logging_obj=logging_obj,
+            stream_options=data.get("stream_options", None),
             _response_headers=process_azure_headers(headers),
         )
         return streamwrapper
@@ -1006,6 +1008,7 @@ async def async_streaming(
                 model=model,
                 custom_llm_provider="azure",
                 logging_obj=logging_obj,
+                stream_options=data.get("stream_options", None),
                 _response_headers=headers,
             )
             return streamwrapper  ## DO NOT make this into an async for ... loop, it will yield an async generator, which won't raise errors if the response fails

litellm/llms/OpenAI/completion/utils.py

@@ -0,0 +1,15 @@
+from collections.abc import Iterable
+from typing import List
+
+
+def is_tokens_or_list_of_tokens(value: List):
+    # Check if it's a list of integers (tokens)
+    if isinstance(value, list) and all(isinstance(item, int) for item in value):
+        return True
+    # Check if it's a list of lists of integers (list of tokens)
+    if isinstance(value, list) and all(
+        isinstance(item, list) and all(isinstance(i, int) for i in item)
+        for item in value
+    ):
+        return True
+    return False

litellm/llms/OpenAI/openai.py

@@ -4,7 +4,7 @@
 import time
 import traceback
 import types
-from typing import Any, Callable, Coroutine, Iterable, Literal, Optional, Union
+from typing import Any, Callable, Coroutine, Iterable, Literal, Optional, Union, cast
 
 import httpx
 import openai
@@ -30,8 +30,10 @@
 
 from ...types.llms.openai import *
 from ..base import BaseLLM
+from ..prompt_templates.common_utils import convert_content_list_to_str
 from ..prompt_templates.factory import custom_prompt, prompt_factory
 from .common_utils import drop_params_from_unprocessable_entity_error
+from .completion.utils import is_tokens_or_list_of_tokens
 
 
 class OpenAIError(Exception):
@@ -420,6 +422,35 @@ def get_config(cls):
             and v is not None
         }
 
+    def _transform_prompt(
+        self,
+        messages: Union[List[AllMessageValues], List[OpenAITextCompletionUserMessage]],
+    ) -> AllPromptValues:
+        if len(messages) == 1:  # base case
+            message_content = messages[0].get("content")
+            if (
+                message_content
+                and isinstance(message_content, list)
+                and is_tokens_or_list_of_tokens(message_content)
+            ):
+                openai_prompt: AllPromptValues = cast(AllPromptValues, message_content)
+            else:
+                openai_prompt = ""
+                content = convert_content_list_to_str(
+                    cast(AllMessageValues, messages[0])
+                )
+                openai_prompt += content
+        else:
+            prompt_str_list: List[str] = []
+            for m in messages:
+                try:  # expect list of int/list of list of int to be a 1 message array only.
+                    content = convert_content_list_to_str(cast(AllMessageValues, m))
+                    prompt_str_list.append(content)
+                except Exception as e:
+                    raise e
+            openai_prompt = prompt_str_list
+        return openai_prompt
+
     def convert_to_chat_model_response_object(
         self,
         response_object: Optional[TextCompletionResponse] = None,
@@ -459,6 +490,7 @@ def convert_to_chat_model_response_object(
 
 
 class OpenAIChatCompletion(BaseLLM):
+
     def __init__(self) -> None:
         super().__init__()
 
@@ -1466,7 +1498,9 @@ async def ahealth_check(
         elif mode == "audio_transcription":
             # Get the current directory of the file being run
             pwd = os.path.dirname(os.path.realpath(__file__))
-            file_path = os.path.join(pwd, "../tests/gettysburg.wav")
+            file_path = os.path.join(
+                pwd, "../../../tests/gettysburg.wav"
+            )  # proxy address
             audio_file = open(file_path, "rb")
             completion = await client.audio.transcriptions.with_raw_response.create(
                 file=audio_file,
@@ -1502,6 +1536,8 @@ async def ahealth_check(
 
 
 class OpenAITextCompletion(BaseLLM):
+    openai_text_completion_global_config = OpenAITextCompletionConfig()
+
     def __init__(self) -> None:
         super().__init__()
 
@@ -1518,7 +1554,7 @@ def completion(
         model_response: ModelResponse,
         api_key: str,
         model: str,
-        messages: list,
+        messages: Union[List[AllMessageValues], List[OpenAITextCompletionUserMessage]],
         timeout: float,
         logging_obj: LiteLLMLoggingObj,
         optional_params: dict,
@@ -1531,24 +1567,18 @@ def completion(
         organization: Optional[str] = None,
         headers: Optional[dict] = None,
     ):
-        super().completion()
         try:
             if headers is None:
                 headers = self.validate_environment(api_key=api_key)
             if model is None or messages is None:
                 raise OpenAIError(status_code=422, message="Missing model or messages")
 
-            if (
-                len(messages) > 0
-                and "content" in messages[0]
-                and isinstance(messages[0]["content"], list)
-            ):
-                prompt = messages[0]["content"]
-            else:
-                prompt = [message["content"] for message in messages]  # type: ignore
-
             # don't send max retries to the api, if set
 
+            prompt = self.openai_text_completion_global_config._transform_prompt(
+                messages
+            )
+
             data = {"model": model, "prompt": prompt, **optional_params}
             max_retries = data.pop("max_retries", 2)
             ## LOGGING

litellm/llms/anthropic/chat/handler.py

@@ -551,6 +551,8 @@ def completion(
                     error_response = getattr(e, "response", None)
                     if error_headers is None and error_response:
                         error_headers = getattr(error_response, "headers", None)
+                    if error_response and hasattr(error_response, "text"):
+                        error_text = getattr(error_response, "text", error_text)
                     raise AnthropicError(
                         message=error_text,
                         status_code=status_code,

litellm/llms/anthropic/chat/transformation.py

@@ -6,11 +6,17 @@
 from litellm.types.llms.anthropic import (
     AnthropicMessageRequestBase,
     AnthropicMessagesRequest,
+    AnthropicMessagesTool,
     AnthropicMessagesToolChoice,
     AnthropicSystemMessageContent,
 )
-from litellm.types.llms.openai import AllMessageValues, ChatCompletionSystemMessage
-from litellm.utils import has_tool_call_blocks
+from litellm.types.llms.openai import (
+    AllMessageValues,
+    ChatCompletionSystemMessage,
+    ChatCompletionToolParam,
+    ChatCompletionToolParamFunctionChunk,
+)
+from litellm.utils import add_dummy_tool, has_tool_call_blocks
 
 from ..common_utils import AnthropicError
 
@@ -146,11 +152,16 @@ def map_openai_params(
             and messages is not None
             and has_tool_call_blocks(messages)
         ):
-            raise litellm.UnsupportedParamsError(
-                message="Anthropic doesn't support tool calling without `tools=` param specified. Pass `tools=` param to enable tool calling.",
-                model="",
-                llm_provider="anthropic",
-            )
+            if litellm.modify_params:
+                optional_params["tools"] = add_dummy_tool(
+                    custom_llm_provider="bedrock_converse"
+                )
+            else:
+                raise litellm.UnsupportedParamsError(
+                    message="Anthropic doesn't support tool calling without `tools=` param specified. Pass `tools=` param OR set `litellm.modify_params = True` // `litellm_settings::modify_params: True` to add dummy tool to the request.",
+                    model="",
+                    llm_provider="anthropic",
+                )
 
         return optional_params
 
@@ -266,18 +277,23 @@ def _transform_request(
             if "anthropic-beta" not in headers:
                 # default to v1 of "anthropic-beta"
                 headers["anthropic-beta"] = "tools-2024-05-16"
-
             anthropic_tools = []
             for tool in optional_params["tools"]:
                 if "input_schema" in tool:  # assume in anthropic format
                     anthropic_tools.append(tool)
                 else:  # assume openai tool call
                     new_tool = tool["function"]
-                    new_tool["input_schema"] = new_tool.pop("parameters")  # rename key
+                    parameters = new_tool.pop(
+                        "parameters",
+                        {
+                            "type": "object",
+                            "properties": {},
+                        },
+                    )
+                    new_tool["input_schema"] = parameters  # rename key
                     if "cache_control" in tool:
                         new_tool["cache_control"] = tool["cache_control"]
                     anthropic_tools.append(new_tool)
-
             optional_params["tools"] = anthropic_tools
 
         data = {

litellm/llms/azure_ai/chat/transformation.py

@@ -26,6 +26,7 @@ def get_required_params(self) -> List[ProviderField]:
 
     def _transform_messages(self, messages: List[AllMessageValues]) -> List:
         for message in messages:
-            message = convert_content_list_to_str(message=message)
-
+            texts = convert_content_list_to_str(message=message)
+            if texts:
+                message["content"] = texts
         return messages

litellm/llms/bedrock/chat/converse_transformation.py

@@ -22,7 +22,7 @@
     ChatCompletionToolParamFunctionChunk,
 )
 from litellm.types.utils import ModelResponse, Usage
-from litellm.utils import CustomStreamWrapper, has_tool_call_blocks
+from litellm.utils import CustomStreamWrapper, add_dummy_tool, has_tool_call_blocks
 
 from ...prompt_templates.factory import _bedrock_converse_messages_pt, _bedrock_tools_pt
 from ..common_utils import BedrockError, get_bedrock_tool_name
@@ -213,11 +213,16 @@ def map_openai_params(
             and messages is not None
             and has_tool_call_blocks(messages)
         ):
-            raise litellm.UnsupportedParamsError(
-                message="Anthropic doesn't support tool calling without `tools=` param specified. Pass `tools=` param to enable tool calling.",
-                model="",
-                llm_provider="anthropic",
-            )
+            if litellm.modify_params:
+                optional_params["tools"] = add_dummy_tool(
+                    custom_llm_provider="bedrock_converse"
+                )
+            else:
+                raise litellm.UnsupportedParamsError(
+                    message="Bedrock doesn't support tool calling without `tools=` param specified. Pass `tools=` param OR set `litellm.modify_params = True` // `litellm_settings::modify_params: True` to add dummy tool to the request.",
+                    model="",
+                    llm_provider="bedrock",
+                )
         return optional_params
 
     def _transform_request(

litellm/llms/prompt_templates/common_utils.py

@@ -7,7 +7,7 @@
 from litellm.types.llms.openai import AllMessageValues
 
 
-def convert_content_list_to_str(message: AllMessageValues) -> AllMessageValues:
+def convert_content_list_to_str(message: AllMessageValues) -> str:
     """
     - handles scenario where content is list and not string
     - content list is just text, and no images
@@ -26,7 +26,4 @@ def convert_content_list_to_str(message: AllMessageValues) -> AllMessageValues:
         elif message_content is not None and isinstance(message_content, str):
             texts = message_content
 
-    if texts:
-        message["content"] = texts
-
-    return message
+    return texts

litellm/llms/prompt_templates/factory.py

@@ -2554,7 +2554,10 @@ def _bedrock_tools_pt(tools: List) -> List[BedrockToolBlock]:
     """
     tool_block_list: List[BedrockToolBlock] = []
     for tool in tools:
-        parameters = tool.get("function", {}).get("parameters", None)
+        parameters = tool.get("function", {}).get("parameters", {
+            "type": "object",
+            "properties": {}
+        })
         name = tool.get("function", {}).get("name", "")
 
         # related issue: https://github.com/BerriAI/litellm/issues/5007

litellm/llms/together_ai/completion/handler.py

@@ -0,0 +1,61 @@
+"""
+Support for OpenAI's `/v1/completions` endpoint. 
+
+Calls done in OpenAI/openai.py as TogetherAI is openai-compatible.
+
+Docs: https://docs.together.ai/reference/completions-1
+"""
+
+from typing import Any, Callable, List, Optional, Union
+
+from litellm.litellm_core_utils.litellm_logging import Logging
+from litellm.types.llms.openai import AllMessageValues, OpenAITextCompletionUserMessage
+from litellm.utils import ModelResponse
+
+from ...OpenAI.openai import OpenAITextCompletion
+from .transformation import TogetherAITextCompletionConfig
+
+together_ai_text_completion_global_config = TogetherAITextCompletionConfig()
+
+
+class TogetherAITextCompletion(OpenAITextCompletion):
+
+    def completion(
+        self,
+        model_response: ModelResponse,
+        api_key: str,
+        model: str,
+        messages: Union[List[AllMessageValues], List[OpenAITextCompletionUserMessage]],
+        timeout: float,
+        logging_obj: Logging,
+        optional_params: dict,
+        print_verbose: Optional[Callable[..., Any]] = None,
+        api_base: Optional[str] = None,
+        acompletion: bool = False,
+        litellm_params=None,
+        logger_fn=None,
+        client=None,
+        organization: Optional[str] = None,
+        headers: Optional[dict] = None,
+    ):
+        prompt = together_ai_text_completion_global_config._transform_prompt(messages)
+
+        message = OpenAITextCompletionUserMessage(role="user", content=prompt)
+        new_messages = [message]
+        return super().completion(
+            model_response=model_response,
+            api_key=api_key,
+            model=model,
+            messages=new_messages,
+            timeout=timeout,
+            logging_obj=logging_obj,
+            optional_params=optional_params,
+            print_verbose=print_verbose,
+            api_base=api_base,
+            acompletion=acompletion,
+            litellm_params=litellm_params,
+            logger_fn=logger_fn,
+            client=client,
+            organization=organization,
+            headers=headers,
+        )