Smolagents: support ImageContent and AudioContent

docs/guide/smolagents.md

@@ -110,6 +110,10 @@ with MCPAdapt(
 
 This approach achieves the same result but uses MCPAdapt directly with its smolagents adapter.
 
+> [!NOTE]
+> The Audio Content is optional. If you want to use the audio content, you need to install the extra `audio` package:
+> `uv add mcpadapt[smolagents,audio]`
+
 ## Full Working Code Example
 
 You can find a fully working script of this example [here](https://github.com/grll/mcpadapt/blob/main/examples/smolagents_pubmed.py)

pyproject.toml

@@ -7,7 +7,7 @@ requires-python = ">=3.10"
 license = "MIT"
 authors = [{ name = "Guillaume Raille", email = "[email protected]" }]
 dependencies = [
-    "mcp>=1.9.0",
+    "mcp>=1.9.4",
     "jsonref>=1.1.0",
     "python-dotenv>=1.0.1",
     "pydantic>=2.10.6",
@@ -44,17 +44,23 @@ llamaindex = [
 test = [
     "pytest-asyncio>=0.25.2",
     "pytest>=8.3.4",
+    "pytest-datadir>=1.7.2",
     "mcpadapt[langchain]",
     "mcpadapt[smolagents]",
     "mcpadapt[crewai]",
-    "mcpadapt[google-genai]"
+    "mcpadapt[google-genai]",
+    "mcpadapt[audio]",
 ]
 crewai = [
     "crewai>=0.108.0",
 ]
 google-genai = [
     "google-genai>=1.2.0",
 ]
+audio = [
+    "torchaudio>=2.7.1",
+    "soundfile>=0.13.1",
+]
 
 [tool.hatch.version]
 path = "src/mcpadapt/__init__.py"

src/mcpadapt/smolagents_adapter.py

@@ -8,19 +8,26 @@
 >>>     print(tools)
 """
 
-import logging
+import base64
 import keyword
+import logging
 import re
-from typing import Any, Callable, Coroutine
+from io import BytesIO
+from typing import TYPE_CHECKING, Any, Callable, Coroutine, Union
 
 import jsonref  # type: ignore
 import mcp
 import smolagents  # type: ignore
+from smolagents.utils import _is_package_available  # type: ignore
 
 from mcpadapt.core import ToolAdapter
 
 logger = logging.getLogger(__name__)
 
+if TYPE_CHECKING:
+    import torch
+    from PIL.Image import Image as PILImage
+
 
 def _sanitize_function_name(name):
     """
@@ -85,7 +92,9 @@ def __init__(
                 self.is_initialized = True
                 self.skip_forward_signature_validation = True
 
-            def forward(self, *args, **kwargs) -> str:
+            def forward(
+                self, *args, **kwargs
+            ) -> Union[str, "PILImage", "torch.Tensor"]:
                 if len(args) > 0:
                     if len(args) == 1 and isinstance(args[0], dict) and not kwargs:
                         mcp_output = func(args[0])
@@ -104,12 +113,35 @@ def forward(self, *args, **kwargs) -> str:
                         f"tool {self.name} returned multiple content, using the first one"
                     )
 
-                if not isinstance(mcp_output.content[0], mcp.types.TextContent):
-                    raise ValueError(
-                        f"tool {self.name} returned a non-text content: `{type(mcp_output.content[0])}`"
-                    )
+                content = mcp_output.content[0]
+
+                if isinstance(content, mcp.types.TextContent):
+                    return content.text
+
+                if isinstance(content, mcp.types.ImageContent):
+                    from PIL import Image
+
+                    image_data = base64.b64decode(content.data)
+                    image = Image.open(BytesIO(image_data))
+                    return image
+
+                if isinstance(content, mcp.types.AudioContent):
+                    if not _is_package_available("torchaudio"):
+                        raise ValueError(
+                            "Audio content requires the torchaudio package to be installed. "
+                            "Please install it with `uv add mcpadapt[smolagents,audio]`.",
+                        )
+                    else:
+                        import torchaudio  # type: ignore
+
+                        audio_data = base64.b64decode(content.data)
+                        audio_io = BytesIO(audio_data)
+                        audio_tensor, _ = torchaudio.load(audio_io)
+                        return audio_tensor
 
-                return mcp_output.content[0].text  # type: ignore
+                raise ValueError(
+                    f"tool {self.name} returned an unsupported content type: {type(content)}"
+                )
 
         # make sure jsonref are resolved
         input_schema = {
@@ -129,7 +161,7 @@ def forward(self, *args, **kwargs) -> str:
             name=mcp_tool.name,
             description=mcp_tool.description or "",
             inputs=input_schema["properties"],
-            output_type="string",
+            output_type="object",
         )
 
         return tool

tests/test_smolagents_adapter.py

@@ -1,3 +1,4 @@
+from pathlib import Path
 from textwrap import dedent
 
 import pytest
@@ -187,3 +188,74 @@ def echo_tool(text: str) -> str:
         assert len(tools) == 1
         assert tools[0].name == "def_"
         assert tools[0](text="hello") == "Echo: hello"
+
+
+@pytest.fixture
+def shared_datadir():
+    return Path(__file__).parent / "data"
+
+
+def test_image_tool(shared_datadir):
+    mcp_server_script = dedent(
+        f"""
+        import os
+        from mcp.server.fastmcp import FastMCP, Image
+
+        mcp = FastMCP("Image Server")
+
+        @mcp.tool("test_image")
+        def test_image() -> Image:
+            path = os.path.join("{shared_datadir}", "random_image.png")
+            return Image(path=path, format='png')
+
+        mcp.run()
+        """
+    )
+    with MCPAdapt(
+        StdioServerParameters(
+            command="uv", args=["run", "python", "-c", mcp_server_script]
+        ),
+        SmolAgentsAdapter(),
+    ) as tools:
+        from PIL.ImageFile import ImageFile
+
+        assert len(tools) == 1
+        assert tools[0].name == "test_image"
+        image_content = tools[0]()
+        assert isinstance(image_content, ImageFile)
+        assert image_content.size == (256, 256)
+
+
+def test_audio_tool(shared_datadir):
+    mcp_server_script = dedent(
+        f"""
+        import os
+        import base64
+        from mcp.server.fastmcp import FastMCP
+        from mcp.types import AudioContent
+
+        mcp = FastMCP("Audio Server")
+
+        @mcp.tool("test_audio")
+        def test_audio() -> AudioContent:
+            path = os.path.join("{shared_datadir}", "white_noise.wav")
+            with open(path, "rb") as f:
+                wav_bytes = f.read()
+
+            return AudioContent(type="audio", data=base64.b64encode(wav_bytes).decode(), mimeType="audio/wav")
+
+        mcp.run()
+        """
+    )
+    with MCPAdapt(
+        StdioServerParameters(
+            command="uv", args=["run", "python", "-c", mcp_server_script]
+        ),
+        SmolAgentsAdapter(),
+    ) as tools:
+        from torch import Tensor  # type: ignore
+
+        assert len(tools) == 1
+        assert tools[0].name == "test_audio"
+        audio_content = tools[0]()
+        assert isinstance(audio_content, Tensor)