{ "leaderboardData": [ { "info": { "name": "Qwen2.5-Omni", "size": "7B", "date": "2025-03-27", "type": "open_source_olm", "source_type": "Self-reported", "source_link": "https://github.com/QwenLM/Qwen2.5-Omni", "link": "https://github.com/QwenLM/Qwen2.5-Omni" }, "image-audio": { "speech": 55.25, "sound event": 60.00, "music": 52.83, "action and activity": "-", "story description": "-", "plot inference": "-", "object identification and description": "-", "contextual and environmental questions": "-", "identity and relationship": "-", "text and symbols": "-", "count and quantity": "-", "overall": 56.13 }, "textual_image": { "speech": "-", "sound event": "-", "music": "-", "action and activity": "-", "story description": "-", "plot inference": "-", "object identification and description": "-", "contextual and environmental questions": "-", "identity and relationship": "-", "text and symbols": "-", "count and quantity": "-", "overall": "-" }, "textual_audio": { "speech": "-", "sound event": "-", "music": "-", "action and activity": "-", "story description": "-", "plot inference": "-", "object identification and description": "-", "contextual and environmental questions": "-", "identity and relationship": "-", "text and symbols": "-", "count and quantity": "-", "overall": "-" }, "textual_image-audio": { "speech": "-", "sound event": "-", "music": "-", "action and activity": "-", "story description": "-", "plot inference": "-", "object identification and description": "-", "contextual and environmental questions": "-", "identity and relationship": "-", "text and symbols": "-", "count and quantity": "-", "overall": "-" } }, { "info": { "name": "OpenOmni", "size": "7B", "date": "2025-01-08", "type": "open_source_olm", "source_type": "Self-reported", "source_link": "https://arxiv.org/abs/2501.04561", "link": "https://arxiv.org/abs/2501.04561" }, "image-audio": { "speech": "-", "sound event": "-", "music": "-", "action and activity": 36.65, "story description": 45.65, "plot inference": 32.91, "object identification and description": 44.08, "contextual and environmental questions": 48.23, "identity and relationship": 34.38, "text and symbols": 24.00, "count and quantity":33.33, "overall": 37.40 }, "textual_image": { "speech": "-", "sound event": "-", "music": "-", "action and activity": "-", "story description": "-", "plot inference": "-", "object identification and description": "-", "contextual and environmental questions": "-", "identity and relationship": "-", "text and symbols": "-", "count and quantity": "-", "overall": "-" }, "textual_audio": { "speech": "-", "sound event": "-", "music": "-", "action and activity": "-", "story description": "-", "plot inference": "-", "object identification and description": "-", "contextual and environmental questions": "-", "identity and relationship": "-", "text and symbols": "-", "count and quantity": "-", "overall": "-" }, "textual_image-audio": { "speech": "-", "sound event": "-", "music": "-", "action and activity": "-", "story description": "-", "plot inference": "-", "object identification and description": "-", "contextual and environmental questions": "-", "identity and relationship": "-", "text and symbols": "-", "count and quantity": "-", "overall": "-" } }, { "info": { "name": "MiniCPM-o 2.6", "size": "7B", "date": "2025-01-26", "type": "open_source_olm", "source_type": "Third-party", "source_link": "https://github.com/baichuan-inc/Baichuan-Omni-1.5", "link": "https://github.com/OpenBMB/MiniCPM-o" }, "image-audio": { "speech": "-", "sound event": "-", "music": "-", "action and activity": "-", "story description": "-", "plot inference": "-", "object identification and description": "-", "contextual and environmental questions": "-", "identity and relationship": "-", "text and symbols": "-", "count and quantity": "-", "overall": 40.5 }, "textual_image": { "speech": "-", "sound event": "-", "music": "-", "action and activity": "-", "story description": "-", "plot inference": "-", "object identification and description": "-", "contextual and environmental questions": "-", "identity and relationship": "-", "text and symbols": "-", "count and quantity": "-", "overall": 30.8 }, "textual_audio": { "speech": "-", "sound event": "-", "music": "-", "action and activity": "-", "story description": "-", "plot inference": "-", "object identification and description": "-", "contextual and environmental questions": "-", "identity and relationship": "-", "text and symbols": "-", "count and quantity": "-", "overall": 53.2 }, "textual_image-audio": { "speech": "-", "sound event": "-", "music": "-", "action and activity": "-", "story description": "-", "plot inference": "-", "object identification and description": "-", "contextual and environmental questions": "-", "identity and relationship": "-", "text and symbols": "-", "count and quantity": "-", "overall": 46.3 } }, { "info": { "name": "Baichuan-Omni", "size": "7B", "date": "2025-01-26", "type": "proprietary_olm", "source_type": "Self-reported", "source_link": "https://github.com/baichuan-inc/Baichuan-Omni-1.5", "link": "https://github.com/OpenBMB/MiniCPM-o" }, "image-audio": { "speech": "-", "sound event": "-", "music": "-", "action and activity": "-", "story description": "-", "plot inference": "-", "object identification and description": "-", "contextual and environmental questions": "-", "identity and relationship": "-", "text and symbols": "-", "count and quantity": "-", "overall": 32.2 }, "textual_image": { "speech": "-", "sound event": "-", "music": "-", "action and activity": "-", "story description": "-", "plot inference": "-", "object identification and description": "-", "contextual and environmental questions": "-", "identity and relationship": "-", "text and symbols": "-", "count and quantity": "-", "overall": 26.5 }, "textual_audio": { "speech": "-", "sound event": "-", "music": "-", "action and activity": "-", "story description": "-", "plot inference": "-", "object identification and description": "-", "contextual and environmental questions": "-", "identity and relationship": "-", "text and symbols": "-", "count and quantity": "-", "overall": 42.6 }, "textual_image-audio": { "speech": "-", "sound event": "-", "music": "-", "action and activity": "-", "story description": "-", "plot inference": "-", "object identification and description": "-", "contextual and environmental questions": "-", "identity and relationship": "-", "text and symbols": "-", "count and quantity": "-", "overall": 44.2 } }, { "info": { "name": "Baichuan-Omni-1.5", "size": "7B", "date": "2025-01-26", "type": "open_source_olm", "source_type": "Self-reported", "source_link": "https://github.com/baichuan-inc/Baichuan-Omni-1.5", "link": "https://github.com/baichuan-inc/Baichuan-Omni-1.5" }, "image-audio": { "speech": "-", "sound event": "-", "music": "-", "action and activity": "-", "story description": "-", "plot inference": "-", "object identification and description": "-", "contextual and environmental questions": "-", "identity and relationship": "-", "text and symbols": "-", "count and quantity": "-", "overall": 42.9 }, "textual_image": { "speech": "-", "sound event": "-", "music": "-", "action and activity": "-", "story description": "-", "plot inference": "-", "object identification and description": "-", "contextual and environmental questions": "-", "identity and relationship": "-", "text and symbols": "-", "count and quantity": "-", "overall": 37.7 }, "textual_audio": { "speech": "-", "sound event": "-", "music": "-", "action and activity": "-", "story description": "-", "plot inference": "-", "object identification and description": "-", "contextual and environmental questions": "-", "identity and relationship": "-", "text and symbols": "-", "count and quantity": "-", "overall": 47.9 }, "textual_image-audio": { "speech": "-", "sound event": "-", "music": "-", "action and activity": "-", "story description": "-", "plot inference": "-", "object identification and description": "-", "contextual and environmental questions": "-", "identity and relationship": "-", "text and symbols": "-", "count and quantity": "-", "overall": 46.9 } }, { "info": { "name": "VITA-1.5", "size": "7B", "date": "2025-01-26", "type": "open_source_olm", "source_type": "Third-party", "source_link": "https://github.com/baichuan-inc/Baichuan-Omni-1.5", "link": "https://github.com/VITA-MLLM/VITA" }, "image-audio": { "speech": "-", "sound event": "-", "music": "-", "action and activity": "-", "story description": "-", "plot inference": "-", "object identification and description": "-", "contextual and environmental questions": "-", "identity and relationship": "-", "text and symbols": "-", "count and quantity": "-", "overall": 33.4 }, "textual_image": { "speech": "-", "sound event": "-", "music": "-", "action and activity": "-", "story description": "-", "plot inference": "-", "object identification and description": "-", "contextual and environmental questions": "-", "identity and relationship": "-", "text and symbols": "-", "count and quantity": "-", "overall": 29.6 }, "textual_audio": { "speech": "-", "sound event": "-", "music": "-", "action and activity": "-", "story description": "-", "plot inference": "-", "object identification and description": "-", "contextual and environmental questions": "-", "identity and relationship": "-", "text and symbols": "-", "count and quantity": "-", "overall": 48.5 }, "textual_image-audio": { "speech": "-", "sound event": "-", "music": "-", "action and activity": "-", "story description": "-", "plot inference": "-", "object identification and description": "-", "contextual and environmental questions": "-", "identity and relationship": "-", "text and symbols": "-", "count and quantity": "-", "overall": 47.2 } }, { "info": { "name": "video-Salmonn", "size": "13B", "date": "2024-09-22", "type": "open_source_olm", "source_type": "OmniBench", "source_link": "https://arxiv.org/abs/2409.15272", "link": "https://arxiv.org/pdf/2406.15704" }, "image-audio": { "speech": 34.11, "sound_event": 31.7, "music": 56.6, "overall": 35.64, "action_and_activity": 31.47, "story_description": 28.26, "plot_inference": 25.74, "object_identification_and_description": 62.56, "contextual_and_environmental_questions": 36.88, "identity_and_relationship": 37.5, "text_and_symbols": 20.0, "count_and_quantity": 6.67 }, "textual_image": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" }, "textual_audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" }, "textual_image-audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" } }, { "info": { "name": "VITA", "size": "8x7B", "date": "2024-10-14", "type": "open_source_olm", "source_type": "Self-reported", "source_link": "https://huggingface.co/multimodal-art-projection/VITA", "link": "https://arxiv.org/abs/2409.15272" }, "image-audio": { "speech": 31.52, "sound event": 32.45, "music": 46.23, "action and activity": 35.86, "story description": 33.04, "plot inference": "29.54%", "object identification and description": 33.65, "contextual and environmental questions": 41.84, "identity and relationship": 21.88, "text and symbols": 16.00, "count and quantity": 6.67, "overall": 33.10 }, "textual_image": { "speech": 31.26, "sound event": 30.94, "music": 37.74, "action and activity": 31.87, "story description": 30.0, "plot inference": 29.96, "object identification and description": 34.12, "contextual and environmental questions": 41.13, "identity and relationship": 28.12, "text and symbols": 12.0, "count and quantity": 6.67, "overall": 31.79 }, "textual_audio": { "speech": 38.0, "sound event": 44.53, "music": 65.09, "action and activity": 51.39, "story description": 43.04, "plot inference": 39.24, "object identification and description": 36.49, "contextual and environmental questions": 46.1, "identity and relationship": 31.25, "text and symbols": 12.0, "count and quantity": 26.67, "overall": 42.03 }, "textual_image-audio": { "speech": 40.86, "sound event": 46.04, "music": 64.15, "action and activity": 51.39, "story description": 44.78, "plot inference": 43.04, "object identification and description": 36.97, "contextual and environmental questions": 51.77, "identity and relationship": 37.5, "text and symbols": 16.0, "count and quantity": 26.67, "overall": 44.22 } }, { "info": { "name": "AnyGPT", "size": "7B", "date": "2024-09-22", "type": "open_source_olm", "source_type": "OmniBench", "source_link": "https://arxiv.org/abs/2409.15272", "link": "https://junzhan2000.github.io/AnyGPT.github.io/" }, "image-audio": { "speech": 17.77, "sound_event": 20.75, "music": 13.21, "overall": 18.04, "action_and_activity": 19.52, "story_description": 16.52, "plot_inference": 14.77, "object_identification_and_description": 22.27, "contextual_and_environmental_questions": 15.6, "identity_and_relationship": 21.88, "text_and_symbols": 12.0, "count_and_quantity": 33.33 }, "textual_image": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" }, "textual_audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" }, "textual_image-audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" } }, { "info": { "name": "UnifiedIO2-large", "size": "1.1B", "date": "2024-09-22", "type": "open_source_olm", "source_type": "OmniBench", "source_link": "https://arxiv.org/abs/2409.15272", "link": "https://github.com/allenai/unified-io-2" }, "image-audio": { "speech": 25.94, "sound_event": 29.06, "music": 30.19, "overall": 27.06, "action_and_activity": 29.88, "story_description": 20.87, "plot_inference": 31.65, "object_identification_and_description": 30.81, "contextual_and_environmental_questions": 23.4, "identity_and_relationship": 18.75, "text_and_symbols": 24.0, "count_and_quantity": 6.67 }, "textual_image": { "speech": 28.4, "sound_event": 32.45, "music": 26.42, "overall": 29.16, "action_and_activity": 28.29, "story_description": 30.43, "plot_inference": 32.91, "object_identification_and_description": 29.38, "contextual_and_environmental_questions": 29.79, "identity_and_relationship": 15.62, "text_and_symbols": 12.0, "count_and_quantity": 13.33 }, "textual_audio": { "speech": 32.56, "sound_event": 33.96, "music": 48.11, "overall": 34.33, "action_and_activity": 34.26, "story_description": 31.74, "plot_inference": 37.13, "object_identification_and_description": 33.65, "contextual_and_environmental_questions": 39.01, "identity_and_relationship": 15.62, "text_and_symbols": 32.0, "count_and_quantity": 40.0 }, "textual_image-audio": { "speech": 30.61, "sound_event": 30.19, "music": 33.02, "overall": 30.74, "action_and_activity": 31.08, "story_description": 31.3, "plot_inference": 37.55, "object_identification_and_description": 27.01, "contextual_and_environmental_questions": 29.79, "identity_and_relationship": 21.88, "text_and_symbols": 16.0, "count_and_quantity": 13.33 } }, { "info": { "name": "UnifiedIO2-xlarge", "size": "3.2B", "date": "2024-09-22", "type": "open_source_olm", "source_type": "OmniBench", "source_link": "https://arxiv.org/abs/2409.15272", "link": "https://github.com/allenai/unified-io-2" }, "image-audio": { "speech": 39.56, "sound_event": 36.98, "music": 29.25, "overall": 38.0, "action_and_activity": 32.27, "story_description": 33.48, "plot_inference": 31.65, "object_identification_and_description": 63.03, "contextual_and_environmental_questions": 34.04, "identity_and_relationship": 34.38, "text_and_symbols": 24.0, "count_and_quantity": 20.0 }, "textual_image": { "speech": 32.43, "sound_event": 32.45, "music": 30.19, "overall": 32.22, "action_and_activity": 33.47, "story_description": 30.0, "plot_inference": 31.65, "object_identification_and_description": 38.86, "contextual_and_environmental_questions": 31.91, "identity_and_relationship": 21.88, "text_and_symbols": 12.0, "count_and_quantity": 20.0 }, "textual_audio": { "speech": 44.49, "sound_event": 38.11, "music": 46.23, "overall": 43.17, "action_and_activity": 35.46, "story_description": 40.43, "plot_inference": 36.29, "object_identification_and_description": 64.93, "contextual_and_environmental_questions": 47.52, "identity_and_relationship": 31.25, "text_and_symbols": 36.0, "count_and_quantity": 13.33 }, "textual_image-audio": { "speech": 31.26, "sound_event": 40.0, "music": 36.79, "overall": 33.8, "action_and_activity": 37.45, "story_description": 30.43, "plot_inference": 30.38, "object_identification_and_description": 39.81, "contextual_and_environmental_questions": 39.01, "identity_and_relationship": 18.75, "text_and_symbols": 12.0, "count_and_quantity": 13.33 } }, { "info": { "name": "UnifiedIO2-xxlarge", "size": "6.8B", "date": "2024-09-22", "type": "open_source_olm", "source_type": "OmniBench", "source_link": "https://arxiv.org/abs/2409.15272", "link": "https://github.com/allenai/unified-io-2" }, "image-audio": { "speech": 34.24, "sound_event": 36.98, "music": 24.53, "overall": 33.98, "action_and_activity": 32.27, "story_description": 29.13, "plot_inference": 29.96, "object_identification_and_description": 48.82, "contextual_and_environmental_questions": 34.75, "identity_and_relationship": 25.0, "text_and_symbols": 8.0, "count_and_quantity": 46.67 }, "textual_image": { "speech": 31.13, "sound_event": 38.87, "music": 21.7, "overall": 32.05, "action_and_activity": 36.65, "story_description": 28.7, "plot_inference": 34.18, "object_identification_and_description": 28.91, "contextual_and_environmental_questions": 35.46, "identity_and_relationship": 21.88, "text_and_symbols": 20.0, "count_and_quantity": 26.67 }, "textual_audio": { "speech": 37.48, "sound_event": 48.3, "music": 46.23, "overall": 40.81, "action_and_activity": 43.82, "story_description": 38.7, "plot_inference": 35.44, "object_identification_and_description": 51.18, "contextual_and_environmental_questions": 42.55, "identity_and_relationship": 21.88, "text_and_symbols": 12.0, "count_and_quantity": 33.33 }, "textual_image-audio": { "speech": 30.22, "sound_event": 44.53, "music": 36.79, "overall": 34.15, "action_and_activity": 42.63, "story_description": 29.57, "plot_inference": 32.49, "object_identification_and_description": 32.23, "contextual_and_environmental_questions": 39.72, "identity_and_relationship": 18.75, "text_and_symbols": 16.0, "count_and_quantity": 26.67 } }, { "info": { "name": "Gemini-1.5-Pro", "size": "-", "date": "2024-09-22", "type": "proprietary_olm", "source_type": "OmniBench", "source_link": "https://arxiv.org/abs/2409.15272", "link": "https://arxiv.org/abs/2403.05530" }, "image-audio": { "speech": 42.67, "sound_event": 42.26, "music": 46.23, "overall": 42.91, "action_and_activity": 41.83, "story_description": 30.87, "plot_inference": 32.91, "object_identification_and_description": 62.56, "contextual_and_environmental_questions": 60.28, "identity_and_relationship": 31.25, "text_and_symbols": 28.0, "count_and_quantity": 13.33 }, "textual_image": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "38.62", "music": "-", "speech": "-", "sound_event": "-" }, "textual_audio": { "speech": 43.58, "sound_event": 46.04, "music": 46.23, "overall": 44.40, "action_and_activity": 44.22, "story_description": 37.39, "plot_inference": 31.65, "object_identification_and_description": 61.61, "contextual_and_environmental_questions": 57.45, "identity_and_relationship": 31.25, "text_and_symbols": 36.0, "count_and_quantity": 33.33 }, "textual_image-audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "42.03", "music": "-", "speech": "-", "sound_event": "-" } }, { "info": { "name": "GPT-4V-Preview", "size": "-", "date": "2024-09-22", "type": "proprietary_vlmalm", "source_type": "OmniBench", "source_link": "https://arxiv.org/abs/2409.15272", "link": "https://openai.com/index/gpt-4v-system-card/" }, "image-audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" }, "textual_image": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" }, "textual_audio": { "speech": 39.31, "sound_event": 38.91, "music": 47.17, "overall": 38.18, "action_and_activity": 43.27, "story_description": 33.64, "plot_inference": 23.36, "object_identification_and_description": 58.79, "contextual_and_environmental_questions": 53.96, "identity_and_relationship": 20.0, "text_and_symbols": 12.5, "count_and_quantity": 20.0 }, "textual_image-audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "33.27", "music": "-", "speech": "-", "sound_event": "-" } }, { "info": { "name": "GPT-4V-0409", "size": "-", "date": "2024-09-22", "type": "proprietary_vlmalm", "source_type": "OmniBench", "source_link": "https://arxiv.org/abs/2409.15272", "link": "https://openai.com/index/gpt-4v-system-card/" }, "image-audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" }, "textual_image": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" }, "textual_audio": { "speech": 33.68, "sound_event": 28.9, "music": 41.51, "overall": 33.36, "action_and_activity": 40.4, "story_description": 32.46, "plot_inference": 23.31, "object_identification_and_description": 36.23, "contextual_and_environmental_questions": 45.32, "identity_and_relationship": 12.5, "text_and_symbols": 12.0, "count_and_quantity": 13.33 }, "textual_image-audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "29.95", "music": "-", "speech": "-", "sound_event": "-" } }, { "info": { "name": "GPT4-o-mini", "size": "-", "date": "2024-09-22", "type": "proprietary_vlmalm", "source_type": "OmniBench", "source_link": "https://arxiv.org/abs/2409.15272", "link": "https://platform.openai.com/docs/overview/" }, "image-audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" }, "textual_image": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" }, "textual_audio": { "speech": 46.26, "sound_event": 46.59, "music": 73.58, "overall": 49.04, "action_and_activity": 42.4, "story_description": 38.43, "plot_inference": 37.71, "object_identification_and_description": 79.23, "contextual_and_environmental_questions": 61.15, "identity_and_relationship": 34.38, "text_and_symbols": 28.0, "count_and_quantity": 26.67 }, "textual_image-audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "51.05", "music": "-", "speech": "-", "sound_event": "-" } }, { "info": { "name": "GPT4-o (0806)", "size": "-", "date": "2024-09-22", "type": "proprietary_vlmalm", "source_type": "OmniBench", "source_link": "https://arxiv.org/abs/2409.15272", "link": "https://platform.openai.com/docs/overview/" }, "image-audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" }, "textual_image": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" }, "textual_audio": { "speech": 50.0, "sound_event": 45.28, "music": 74.53, "overall": 51.14, "action_and_activity": 46.61, "story_description": 44.78, "plot_inference": 43.46, "object_identification_and_description": 66.82, "contextual_and_environmental_questions": 67.86, "identity_and_relationship": 28.12, "text_and_symbols": 40.0, "count_and_quantity": 40.0 }, "textual_image-audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "53.77", "music": "-", "speech": "-", "sound_event": "-" } }, { "info": { "name": "Claude-3.5-Sonnet", "size": "-", "date": "2024-09-22", "type": "proprietary_vlmalm", "source_type": "OmniBench", "source_link": "https://arxiv.org/abs/2409.15272", "link": "https://platform.openai.com/docs/overview/" }, "image-audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" }, "textual_image": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" }, "textual_audio": { "speech": 57.07, "sound_event": 63.77, "music": 65.09, "overall": 59.37, "action_and_activity": 64.14, "story_description": 45.65, "plot_inference": 55.7, "object_identification_and_description": 70.14, "contextual_and_environmental_questions": 72.34, "identity_and_relationship": 40.62, "text_and_symbols": 36.0, "count_and_quantity": 53.33 }, "textual_image-audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "56.83", "music": "-", "speech": "-", "sound_event": "-" } }, { "info": { "name": "Reka-core-20240501", "size": "-", "date": "2024-09-22", "type": "proprietary_olm", "source_type": "OmniBench", "source_link": "https://arxiv.org/abs/2409.15272", "link": "https://www.reka.ai/ourmodels/" }, "image-audio": { "speech": 31.89, "sound_event": 26.04, "music": 33.02, "overall": 30.39, "action_and_activity": 25.5, "story_description": 24.78, "plot_inference": 20.68, "object_identification_and_description": 49.76, "contextual_and_environmental_questions": 39.01, "identity_and_relationship": 28.12, "text_and_symbols": 41.18, "count_and_quantity": 7.14 }, "textual_image": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "29.42", "music": "-", "speech": "-", "sound_event": "-" }, "textual_audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "46.58", "music": "-", "speech": "-", "sound_event": "-" }, "textual_image-audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "42.23", "music": "-", "speech": "-", "sound_event": "-" } }, { "info": { "name": "InternVL-2", "size": "2B", "date": "2024-09-22", "type": "open_source_vlmalm", "source_type": "OmniBench", "source_link": "https://arxiv.org/abs/2409.15272", "link": "https://internvl.github.io/blog/2024-07-02-InternVL-2.0/" }, "image-audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" }, "textual_image": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" }, "textual_audio": { "speech": 38.0, "sound_event": 49.43, "music": 55.66, "overall": 42.29, "action_and_activity": 52.19, "story_description": 42.61, "plot_inference": 41.77, "object_identification_and_description": 35.07, "contextual_and_environmental_questions": 43.97, "identity_and_relationship": 21.88, "text_and_symbols": 28.0, "count_and_quantity": 33.33 }, "textual_image-audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" } }, { "info": { "name": "InternVL-2", "size": "8B", "date": "2024-09-22", "type": "open_source_vlmalm", "source_type": "OmniBench", "source_link": "https://arxiv.org/abs/2409.15272", "link": "https://internvl.github.io/blog/2024-07-02-InternVL-2.0/" }, "image-audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" }, "textual_image": { "speech": 44.62, "sound_event": 49.43, "music": 64.15, "overall": 47.55, "action_and_activity": 56.18, "story_description": 47.39, "plot_inference": 50.21, "object_identification_and_description": 38.86, "contextual_and_environmental_questions": 48.23, "identity_and_relationship": 25.0, "text_and_symbols": 40.0, "count_and_quantity": 40.0 }, "textual_audio": { "speech": 46.17, "sound_event": 56.98, "music": 68.87, "overall": 50.79, "action_and_activity": 61.35, "story_description": 50.0, "plot_inference": 54.85, "object_identification_and_description": 39.34, "contextual_and_environmental_questions": 55.32, "identity_and_relationship": 25.0, "text_and_symbols": 36.0, "count_and_quantity": 20.0 }, "textual_image-audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" } }, { "info": { "name": "InternVL-2", "size": "26B", "date": "2024-09-22", "type": "open_source_vlmalm", "source_type": "OmniBench", "source_link": "https://arxiv.org/abs/2409.15272", "link": "https://internvl.github.io/blog/2024-07-02-InternVL-2.0/" }, "image-audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" }, "textual_image": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" }, "textual_audio": { "speech": 47.08, "sound_event": 58.87, "music": 67.92, "overall": 51.75, "action_and_activity": 64.54, "story_description": 48.26, "plot_inference": 52.32, "object_identification_and_description": 40.76, "contextual_and_environmental_questions": 60.28, "identity_and_relationship": 25.0, "text_and_symbols": 40.0, "count_and_quantity": 33.33 }, "textual_image-audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" } }, { "info": { "name": "InternVL-2", "size": "40B", "date": "2024-09-22", "type": "open_source_vlmalm", "source_type": "OmniBench", "source_link": "https://arxiv.org/abs/2409.15272", "link": "https://internvl.github.io/blog/2024-07-02-InternVL-2.0/" }, "image-audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" }, "textual_image": { "speech": 43.06, "sound_event": 50.19, "music": 69.81, "overall": 47.2, "action_and_activity": 52.59, "story_description": 44.35, "plot_inference": 51.48, "object_identification_and_description": 37.91, "contextual_and_environmental_questions": 53.9, "identity_and_relationship": 25.0, "text_and_symbols": 44.0, "count_and_quantity": 53.33 }, "textual_audio": { "speech": 50.32, "sound_event": 59.62, "music": 69.81, "overall": 54.29, "action_and_activity": 64.54, "story_description": 49.13, "plot_inference": 61.6, "object_identification_and_description": 36.97, "contextual_and_environmental_questions": 65.96, "identity_and_relationship": 31.25, "text_and_symbols": 40.0, "count_and_quantity": 53.33 }, "textual_image-audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" } }, { "info": { "name": "Idefics2", "size": "8B", "date": "2024-09-22", "type": "open_source_vlmalm", "source_type": "OmniBench", "source_link": "https://arxiv.org/abs/2409.15272", "link": "https://huggingface.co/blog/idefics2" }, "image-audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" }, "textual_image": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" }, "textual_audio": { "speech": 39.95, "sound_event": 53.21, "music": 62.26, "overall": 45.1, "action_and_activity": 54.98, "story_description": 43.04, "plot_inference": 47.68, "object_identification_and_description": 31.28, "contextual_and_environmental_questions": 55.32, "identity_and_relationship": 25.0, "text_and_symbols": 24.0, "count_and_quantity": 46.67 }, "textual_image-audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" } }, { "info": { "name": "Mantis-Idefics2", "size": "8B", "date": "2024-09-22", "type": "open_source_vlmalm", "source_type": "OmniBench", "source_link": "https://arxiv.org/abs/2409.15272", "link": "https://tiger-ai-lab.github.io/Mantis/" }, "image-audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" }, "textual_image": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" }, "textual_audio": { "speech": 42.41, "sound_event": 52.83, "music": 56.6, "overall": 46.15, "action_and_activity": 49.8, "story_description": 37.39, "plot_inference": 44.73, "object_identification_and_description": 50.71, "contextual_and_environmental_questions": 58.16, "identity_and_relationship": 25.0, "text_and_symbols": 28.0, "count_and_quantity": 40.0 }, "textual_image-audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" } }, { "info": { "name": "XComposer2-4KHD", "size": "7B", "date": "2024-09-22", "type": "open_source_vlmalm", "source_type": "OmniBench", "source_link": "https://arxiv.org/abs/2409.15272", "link": "https://arxiv.org/abs/2404.06512/" }, "image-audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" }, "textual_image": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" }, "textual_audio": { "speech": 38.65, "sound_event": 50.19, "music": 66.98, "overall": 43.96, "action_and_activity": 54.18, "story_description": 40.0, "plot_inference": 42.62, "object_identification_and_description": 36.49, "contextual_and_environmental_questions": 52.48, "identity_and_relationship": 18.75, "text_and_symbols": 36.0, "count_and_quantity": 46.67 }, "textual_image-audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" } }, { "info": { "name": "Deepseek-VL-Chat", "size": "7B", "date": "2024-09-22", "type": "open_source_vlmalm", "source_type": "OmniBench", "source_link": "https://arxiv.org/abs/2409.15272", "link": "https://huggingface.co/deepseek-ai/deepseek-vl-7b-chat" }, "image-audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" }, "textual_image": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" }, "textual_audio": { "speech": 36.58, "sound_event": 41.51, "music": 57.55, "overall": 39.67, "action_and_activity": 39.04, "story_description": 36.52, "plot_inference": 41.35, "object_identification_and_description": 36.02, "contextual_and_environmental_questions": 53.9, "identity_and_relationship": 31.25, "text_and_symbols": 32.0, "count_and_quantity": 20.0 }, "textual_image-audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" } }, { "info": { "name": "Cambrian", "size": "8B", "date": "2024-09-22", "type": "open_source_vlmalm", "source_type": "OmniBench", "source_link": "https://arxiv.org/abs/2409.15272", "link": "https://cambrian-mllm.github.io/" }, "image-audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" }, "textual_image": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" }, "textual_audio": { "speech": 38.91, "sound_event": 43.77, "music": 61.32, "overall": 42.12, "action_and_activity": 48.61, "story_description": 45.22, "plot_inference": 38.82, "object_identification_and_description": 37.44, "contextual_and_environmental_questions": 51.77, "identity_and_relationship": 12.5, "text_and_symbols": 20.0, "count_and_quantity": 13.33 }, "textual_image-audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" } }, { "info": { "name": "Cambrian", "size": "13B", "date": "2024-09-22", "type": "open_source_vlmalm", "source_type": "OmniBench", "source_link": "https://arxiv.org/abs/2409.15272", "link": "https://cambrian-mllm.github.io/" }, "image-audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" }, "textual_image": { "speech": 39.3, "sound_event": 46.04, "music": 63.21, "overall": 43.08, "action_and_activity": 48.61, "story_description": 44.35, "plot_inference": 44.3, "object_identification_and_description": 35.07, "contextual_and_environmental_questions": 51.77, "identity_and_relationship": 25.0, "text_and_symbols": 16.0, "count_and_quantity": 26.67 }, "textual_audio": { "speech": 41.5, "sound_event": 48.68, "music": 61.32, "overall": 45.01, "action_and_activity": 52.99, "story_description": 46.52, "plot_inference": 45.99, "object_identification_and_description": 36.02, "contextual_and_environmental_questions": 50.35, "identity_and_relationship": 18.75, "text_and_symbols": 32.0, "count_and_quantity": 26.67 }, "textual_image-audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" } }, { "info": { "name": "Cambrian", "size": "34B", "date": "2024-09-22", "type": "open_source_vlmalm", "source_type": "OmniBench", "source_link": "https://arxiv.org/abs/2409.15272", "link": "https://cambrian-mllm.github.io/" }, "image-audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" }, "textual_image": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" }, "textual_audio": { "speech": 42.93, "sound_event": 47.92, "music": 71.7, "overall": 46.76, "action_and_activity": 54.58, "story_description": 43.04, "plot_inference": 47.26, "object_identification_and_description": 38.86, "contextual_and_environmental_questions": 59.57, "identity_and_relationship": 21.88, "text_and_symbols": 40.0, "count_and_quantity": 20.0 }, "textual_image-audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" } }, { "info": { "name": "Qwen2-VL-Chat", "size": "2B", "date": "2024-09-22", "type": "open_source_vlmalm", "source_type": "OmniBench", "source_link": "https://arxiv.org/abs/2409.15272", "link": "https://arxiv.org/abs/2409.12191" }, "image-audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" }, "textual_image": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" }, "textual_audio": { "speech": 40.21, "sound_event": 38.49, "music": 68.87, "overall": 42.47, "action_and_activity": 43.03, "story_description": 43.04, "plot_inference": 44.73, "object_identification_and_description": 37.44, "contextual_and_environmental_questions": 46.1, "identity_and_relationship": 31.25, "text_and_symbols": 36.0, "count_and_quantity": 60.0 }, "textual_image-audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" } }, { "info": { "name": "Qwen2-VL-Chat", "size": "7B", "date": "2024-09-22", "type": "open_source_vlmalm", "source_type": "OmniBench", "source_link": "https://arxiv.org/abs/2409.15272", "link": "https://arxiv.org/abs/2409.12191" }, "image-audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" }, "textual_image": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" }, "textual_audio": { "speech": 42.93, "sound_event": 56.6, "music": 69.81, "overall": 48.6, "action_and_activity": 60.96, "story_description": 43.48, "plot_inference": 51.48, "object_identification_and_description": 36.97, "contextual_and_environmental_questions": 56.03, "identity_and_relationship": 34.38, "text_and_symbols": 24.0, "count_and_quantity": 40.0 }, "textual_image-audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" } }, { "info": { "name": "LLaVA-OneVision", "size": "0.5B", "date": "2024-09-22", "type": "open_source_vlmalm", "source_type": "OmniBench", "source_link": "https://arxiv.org/abs/2409.15272", "link": "https://arxiv.org/abs/2408.03326" }, "image-audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" }, "textual_image": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" }, "textual_audio": { "speech": 35.15, "sound_event": 39.62, "music": 54.72, "overall": 38.0, "action_and_activity": 34.66, "story_description": 42.17, "plot_inference": 41.35, "object_identification_and_description": 36.49, "contextual_and_environmental_questions": 42.55, "identity_and_relationship": 21.88, "text_and_symbols": 24.0, "count_and_quantity": 13.33 }, "textual_image-audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" } }, { "info": { "name": "LLaVA-OneVision", "size": "7B", "date": "2024-09-22", "type": "open_source_vlmalm", "source_type": "OmniBench", "source_link": "https://arxiv.org/abs/2409.15272", "link": "https://arxiv.org/abs/2408.03326" }, "image-audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" }, "textual_image": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" }, "textual_audio": { "speech": 44.36, "sound_event": 49.43, "music": 60.38, "overall": 47.02, "action_and_activity": 56.57, "story_description": 43.91, "plot_inference": 50.63, "object_identification_and_description": 33.18, "contextual_and_environmental_questions": 60.28, "identity_and_relationship": 21.88, "text_and_symbols": 28.0, "count_and_quantity": 33.33 }, "textual_image-audio": { "action_and_activity": "-", "story_description": "-", "plot_inference": "-", "object_identification_and_description": "-", "contextual_and_environmental_questions": "-", "identity_and_relationship": "-", "text_and_symbols": "-", "count_and_quantity": "-", "overall": "-", "music": "-", "speech": "-", "sound_event": "-" } } ] }