{"id":"https://openalex.org/W4403789506","doi":"https://doi.org/10.48550/arxiv.2409.18216","title":"MMMT-IF: A Challenging Multimodal Multi-Turn Instruction Following Benchmark","display_name":"MMMT-IF: A Challenging Multimodal Multi-Turn Instruction Following Benchmark","publication_year":2024,"publication_date":"2024-09-26","ids":{"openalex":"https://openalex.org/W4403789506","doi":"https://doi.org/10.48550/arxiv.2409.18216"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2409.18216","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2409.18216","pdf_url":"https://arxiv.org/pdf/2409.18216","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2409.18216","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Epstein, Elliot L.","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Epstein, Elliot L.","raw_affiliation_strings":["Stanford University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103119755","display_name":"Kaisheng Yao","orcid":"https://orcid.org/0000-0002-8949-9367","observed_orcids":["https://orcid.org/0000-0002-8949-9367"]},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yao, Kaisheng","raw_affiliation_strings":["Google"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100683043","display_name":"Jing Li","orcid":"https://orcid.org/0000-0002-2410-5873","observed_orcids":["https://orcid.org/0000-0002-2410-5873"]},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Li, Jing","raw_affiliation_strings":["Google"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018174137","display_name":"Xinyi Bai","orcid":"https://orcid.org/0000-0002-1565-4344","observed_orcids":["https://orcid.org/0000-0002-1565-4344"]},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Bai, Xinyi","raw_affiliation_strings":["Google"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5033846851","display_name":"Hamid Palangi","orcid":"https://orcid.org/0000-0003-2912-4579","observed_orcids":["https://orcid.org/0000-0003-2912-4579"]},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Palangi, Hamid","raw_affiliation_strings":["Google"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google","institution_ids":["https://openalex.org/I1291425158"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":15,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.6699000000953674,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.6699000000953674,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.31369999051094055,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.005100000184029341,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/instruction-following","display_name":"instruction following","score":0.8960000276565552},{"id":"https://openalex.org/keywords/benchmark-dataset","display_name":"benchmark dataset","score":0.5730000138282776},{"id":"https://openalex.org/keywords/llm-based-evaluation","display_name":"LLM-based evaluation","score":0.3479999899864197},{"id":"https://openalex.org/keywords/assessment-bias","display_name":"evaluation bias","score":0.5389999747276306},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.5450000166893005}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7740864753723145},{"id":"https://openalex.org/C85641259","wikidata":"https://www.wikidata.org/wiki/Q290042","display_name":"Turn (biochemistry)","level":2,"score":0.6196311116218567},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5206243991851807},{"id":"https://openalex.org/C2776352735","wikidata":"https://www.wikidata.org/wiki/Q2313343","display_name":"Turn-taking","level":3,"score":0.4585113525390625},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3819662928581238},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3201853632926941},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.2536764442920685},{"id":"https://openalex.org/C46312422","wikidata":"https://www.wikidata.org/wiki/Q11024","display_name":"Communication","level":1,"score":0.14288607239723206},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.13698190450668335},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.11014354228973389},{"id":"https://openalex.org/C58640448","wikidata":"https://www.wikidata.org/wiki/Q42515","display_name":"Cartography","level":1,"score":0.10011222958564758},{"id":"https://openalex.org/C2777200299","wikidata":"https://www.wikidata.org/wiki/Q52943","display_name":"Conversation","level":2,"score":0.0650452971458435},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2409.18216","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2409.18216","pdf_url":"https://arxiv.org/pdf/2409.18216","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2409.18216","doi":"https://doi.org/10.48550/arxiv.2409.18216","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2409.18216","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2409.18216","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2409.18216","pdf_url":"https://arxiv.org/pdf/2409.18216","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4403789506.pdf","grobid_xml":"https://content.openalex.org/works/W4403789506.grobid-xml"},"referenced_works_count":24,"referenced_works":["https://openalex.org/W3135099046","https://openalex.org/W4309395035","https://openalex.org/W4327810158","https://openalex.org/W4385645323","https://openalex.org/W4388718089","https://openalex.org/W4391673253","https://openalex.org/W4391876619","https://openalex.org/W4392616695","https://openalex.org/W4392679398","https://openalex.org/W4393399744","https://openalex.org/W4393969293","https://openalex.org/W4398757454","https://openalex.org/W4399912938","https://openalex.org/W4402667081","https://openalex.org/W4402670102","https://openalex.org/W4402670999","https://openalex.org/W4402684226","https://openalex.org/W4402716477","https://openalex.org/W4403081466","https://openalex.org/W4404783119","https://openalex.org/W4404788931","https://openalex.org/W4406302175","https://openalex.org/W4415798312","https://openalex.org/W4415799083"],"related_works":["https://openalex.org/W4417251337","https://openalex.org/W4404088778","https://openalex.org/W7163885299","https://openalex.org/W4400340507","https://openalex.org/W4403345787","https://openalex.org/W4416425863","https://openalex.org/W4412888403","https://openalex.org/W4399911759","https://openalex.org/W7137876994","https://openalex.org/W7160115305"],"abstract_inverted_index":{"Evaluating":[0],"instruction":[1,73,149],"following":[2,150,245],"capabilities":[3],"for":[4,23,125],"multimodal,":[5],"multi-turn":[6,46],"dialogue":[7],"is":[8,21,193],"challenging.":[9],"With":[10],"potentially":[11],"multiple":[12],"instructions":[13,53,65,76,97,204,213,253],"in":[14,121,244,250,256],"the":[15,19,37,57,85,93,96,117,180,208,212,218,221,225,236,239,246,252,257,265],"input":[16,223],"model":[17,135,222,258],"context,":[18,224],"task":[20,240],"time-consuming":[22],"human":[24,148],"raters":[25],"and":[26,70,162,199,268],"we":[27],"show":[28,157],"LLM":[29],"based":[30,45],"judges":[31],"are":[32,77,99,214],"biased":[33],"towards":[34],"answers":[35],"from":[36,172],"same":[38],"model.":[39],"We":[40,83,260],"propose":[41],"MMMT-IF,":[42],"an":[43],"image":[44],"Q$\\&$A":[47],"evaluation":[48],"set":[49,109],"with":[50,147,238],"added":[51],"global":[52],"between":[54],"questions,":[55],"constraining":[56],"answer":[58],"format.":[59],"This":[60],"challenges":[61],"models":[62],"to":[63,91,182,217,262],"retrieve":[64],"dispersed":[66],"across":[67,179],"long":[68],"dialogues":[69],"reason":[71],"under":[72],"constraints.":[74],"All":[75],"objectively":[78],"verifiable":[79],"through":[80],"code":[81],"execution.":[82],"introduce":[84],"Programmatic":[86],"Instruction":[87],"Following":[88],"($\\operatorname{PIF}$)":[89],"metric":[90,145,169,227,269],"measure":[92],"fraction":[94,118],"of":[95,110,119,132,141,207,220],"that":[98,170,235],"correctly":[100],"followed":[101],"while":[102],"performing":[103],"a":[104,122,138,167],"reasoning":[105],"task.":[106],"The":[107,143],"$\\operatorname{PIF-N-K}$":[108],"metrics":[111],"further":[112],"evaluates":[113],"robustness":[114],"by":[115,229],"measuring":[116],"samples":[120],"corpus":[123],"where,":[124],"each":[126,191],"sample,":[127],"at":[128,176,184],"least":[129],"K":[130],"out":[131,255],"N":[133],"generated":[134],"responses":[136],"achieve":[137],"$\\operatorname{PIF}$":[139,144,168,226],"score":[140],"one.":[142],"aligns":[146],"ratings,":[151],"showing":[152,234],"60":[153],"percent":[154],"correlation.":[155],"Experiments":[156],"Gemini":[158,200],"1.5":[159],"Pro,":[160],"GPT-4o,":[161],"Claude":[163],"3.5":[164],"Sonnet,":[165],"have":[166],"drops":[171],"0.81":[173],"on":[174,232],"average":[175],"turn":[177,185],"1":[178],"models,":[181],"0.64":[183],"20.":[186],"Across":[187],"all":[188,203,211],"turns,":[189],"when":[190],"response":[192],"repeated":[194],"4":[195],"times":[196],"($\\operatorname{PIF-4-4}$),":[197],"GPT-4o":[198],"successfully":[201],"follow":[202],"only":[205,243],"$11\\%$":[206],"time.":[209],"When":[210],"also":[215,249],"appended":[216],"end":[219],"improves":[228],"22.3":[230],"points":[231],"average,":[233],"challenge":[237],"lies":[241],"not":[242],"instructions,":[247],"but":[248],"retrieving":[251],"spread":[254],"context.":[259],"plan":[261],"open":[263],"source":[264],"MMMT-IF":[266],"dataset":[267],"computation":[270],"code.":[271]},"counts_by_year":[{"year":2026,"cited_by_count":10},{"year":2025,"cited_by_count":5}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2024-10-27T00:00:00"}