Skip to content
Merged
Changes from 1 commit
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
8b02948
feat(perf): add typed args for MMMU multi-image data
Bruce-Yii Sep 11, 2026
20b4a57
feat(perf): add real MMMU multi-image dataset mode
Bruce-Yii Sep 11, 2026
9679a8b
feat(perf): register MMMU multi-image dataset
Bruce-Yii Sep 11, 2026
c5be7d6
test(perf): cover MMMU multi-image request construction
Bruce-Yii Sep 11, 2026
30b1d1e
docs(perf): add multi-image stress testing guide
Bruce-Yii Sep 11, 2026
1444d4b
docs(perf): add Chinese multi-image stress testing guide
Bruce-Yii Sep 11, 2026
768a2f2
docs(perf): link multi-image stress testing guide
Bruce-Yii Sep 11, 2026
32b07e8
docs(perf): link Chinese multi-image stress testing guide
Bruce-Yii Sep 11, 2026
09a4f2a
fix(perf): enforce multi-image dataset invariant
Bruce-Yii Sep 11, 2026
43b519e
test(perf): tighten MMMU multi-image invariants
Bruce-Yii Sep 11, 2026
45090d7
docs(perf): align multi-image minimum with dataset contract
Bruce-Yii Sep 11, 2026
84ee9e5
docs(perf): align Chinese multi-image minimum
Bruce-Yii Sep 11, 2026
1b2abe1
test(perf): fix tokenize-guard test setup and class blank line
Sep 11, 2026
a2b2368
fix(perf): normalize MMMU alpha images before JPEG encoding
Bruce-Yii Sep 16, 2026
7f81913
test(perf): cover RGBA MMMU image encoding
Bruce-Yii Sep 16, 2026
62c7925
test(perf): decouple MMMU regression from review history
Yunnglin Sep 17, 2026
75c05e0
docs(perf): clarify multi-image backend compatibility
Yunnglin Sep 17, 2026
54191d1
docs(perf): consolidate multi-image guidance
Yunnglin Sep 17, 2026
8303a32
docs(perf): simplify multi-image headings
Yunnglin Sep 17, 2026
db16b75
docs(perf): centralize multi-image dataset reference
Yunnglin Sep 17, 2026
258a0d8
refactor(perf): fix MMMU multi-image workload
Yunnglin Sep 17, 2026
026248e
feat(perf): load all MMMU multi-image subjects
Yunnglin Sep 17, 2026
555ae32
docs(perf): simplify MMMU dataset description
Yunnglin Sep 17, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
test(perf): cover RGBA MMMU image encoding
  • Loading branch information
Bruce-Yii committed Sep 16, 2026
commit 7f81913caeecbb577fc7fb447853b115a09bb74d
39 changes: 37 additions & 2 deletions tests/perf/test_mmmu_multi_image.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import base64
from io import BytesIO

import pytest
Expand All @@ -17,9 +18,9 @@ def _args(dataset_args=None, **kwargs) -> Arguments:
)


def _image_bytes() -> bytes:
def _image_bytes(mode: str = 'RGB') -> bytes:
buffer = BytesIO()
Image.new('RGB', (2, 2), 'white').save(buffer, format='PNG')
Image.new(mode, (2, 2), 'white').save(buffer, format='PNG')
return buffer.getvalue()


Expand Down Expand Up @@ -60,6 +61,40 @@ def fake_load_hub_dataset(dataset_id, split='train', subset='default'):
assert [part['type'] for part in message['content']] == ['text', 'image_url', 'image_url']
assert all(part['image_url']['url'].startswith('data:image/') for part in message['content'][1:])

def test_encodes_alpha_bearing_images_end_to_end(self, monkeypatch):
# Regression for Yunnglin's blocking review: real AI-ModelScope/MMMU
# Music validation rows contain RGBA images, and the JPEG default of
# PIL_to_base64 cannot write alpha-bearing modes. The first MMMU row
# must produce its request instead of raising mid-iteration.
plugin = MMMUMultiImageDatasetPlugin(_args({'subset': 'Music', 'min_images': 2}))
rows = [
{
'question': 'Which image matches the score?',
'options': "['A', 'B']",
'image_1': Image.new('RGBA', (4, 4), (255, 0, 0, 128)),
'image_2': Image.new('LA', (4, 4), (120, 200)),
'image_3': {'bytes': _image_bytes('RGBA')},
'image_4': Image.new('P', (4, 4)),
'image_5': Image.new('RGB', (4, 4), 'blue'),
}
]
monkeypatch.setattr(plugin, 'load_hub_dataset', lambda **_: rows)

requests = list(plugin.build_messages())

assert len(requests) == 1
message = requests[0][0]
image_parts = [part for part in message['content'] if part['type'] == 'image_url']
# Image order and count are preserved.
assert len(image_parts) == 5
for part in image_parts:
url = part['image_url']['url']
# The data-URL MIME must match the encoded bytes.
assert url.startswith('data:image/jpeg;base64,')
payload = url.split(',', 1)[1]
# The declared JPEG MIME is backed by real JPEG bytes.
assert Image.open(BytesIO(base64.b64decode(payload))).format == 'JPEG'

def test_skips_rows_below_minimum_image_count(self, monkeypatch):
plugin = MMMUMultiImageDatasetPlugin(_args({'min_images': 2}))
monkeypatch.setattr(
Expand Down
Loading