{"id":"https://openalex.org/W4401201041","doi":"https://doi.org/10.48550/arxiv.2407.18961","title":"MMAU: A Holistic Benchmark of Agent Capabilities Across Diverse Domains","display_name":"MMAU: A Holistic Benchmark of Agent Capabilities Across Diverse Domains","publication_year":2024,"publication_date":"2024-07-18","ids":{"openalex":"https://openalex.org/W4401201041","doi":"https://doi.org/10.48550/arxiv.2407.18961"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2407.18961","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2407.18961","pdf_url":"https://arxiv.org/pdf/2407.18961","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2407.18961","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5111279776","display_name":"Guoli Yin","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210153776","display_name":"Apple (United States)","ror":"https://ror.org/059hsda18","country_code":"US","type":"company","lineage":["https://openalex.org/I4210153776"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yin, Guoli","raw_affiliation_strings":["Apple Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Apple Inc","institution_ids":["https://openalex.org/I4210153776"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025733574","display_name":"Haoping Bai","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210153776","display_name":"Apple (United States)","ror":"https://ror.org/059hsda18","country_code":"US","type":"company","lineage":["https://openalex.org/I4210153776"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Bai, Haoping","raw_affiliation_strings":["Apple Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Apple Inc","institution_ids":["https://openalex.org/I4210153776"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Ma, Shuang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210153776","display_name":"Apple (United States)","ror":"https://ror.org/059hsda18","country_code":"US","type":"company","lineage":["https://openalex.org/I4210153776"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ma, Shuang","raw_affiliation_strings":["Apple Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Apple Inc","institution_ids":["https://openalex.org/I4210153776"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Nan, Feng","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nan, Feng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080724634","display_name":"Yanchao Sun","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sun, Yanchao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032228016","display_name":"Zhaoyang Xu","orcid":"https://orcid.org/0000-0002-4118-7639","observed_orcids":["https://orcid.org/0000-0002-4118-7639"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Zhaoyang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064498067","display_name":"Shen Ma","orcid":"https://orcid.org/0000-0001-8460-874X","observed_orcids":["https://orcid.org/0000-0001-8460-874X","https://orcid.org/0009-0000-0105-798X"]},"institutions":[{"id":"https://openalex.org/I4210153776","display_name":"Apple (United States)","ror":"https://ror.org/059hsda18","country_code":"US","type":"company","lineage":["https://openalex.org/I4210153776"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ma, Shen","raw_affiliation_strings":["Apple Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Apple Inc","institution_ids":["https://openalex.org/I4210153776"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100933013","display_name":"Jiarui Lu","orcid":"https://orcid.org/0000-0002-1326-6559","observed_orcids":["https://orcid.org/0000-0002-1326-6559"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lu, Jiarui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024691503","display_name":"Xiang Kong","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kong, Xiang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113315269","display_name":"Aonan Zhang","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Aonan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024743276","display_name":"Dian Ang Yap","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yap, Dian Ang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"zhang, Yizhe","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"zhang, Yizhe","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057983361","display_name":"Karsten Ahnert","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ahnert, Karsten","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046773338","display_name":"Vik Kamath","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kamath, Vik","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5033838234","display_name":"Mathias Berglund","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Berglund, Mathias","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108963721","display_name":"Dominic Walsh","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Walsh, Dominic","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007485736","display_name":"Tobias Gindele","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gindele, Tobias","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108963722","display_name":"Juergen Wiest","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wiest, Juergen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Lai, Zhengfeng","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lai, Zhengfeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100377916","display_name":"Xiaoming Wang","orcid":"https://orcid.org/0000-0003-4559-7413","observed_orcids":["https://orcid.org/0000-0003-4559-7413"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Xiaoming","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068249077","display_name":"Jiulong Shan","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shan, Jiulong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5073266459","display_name":"Meng Cao","orcid":"https://orcid.org/0000-0002-2938-8788","observed_orcids":["https://orcid.org/0000-0002-2938-8788"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cao, Meng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112901893","display_name":"Ruoming Pang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210153776","display_name":"Apple (United States)","ror":"https://ror.org/059hsda18","country_code":"US","type":"company","lineage":["https://openalex.org/I4210153776"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Pang, Ruoming","raw_affiliation_strings":["Apple Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Apple Inc","institution_ids":["https://openalex.org/I4210153776"]}]},{"author_position":"last","author":{"id":null,"display_name":"Wang, Zirui","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210153776","display_name":"Apple (United States)","ror":"https://ror.org/059hsda18","country_code":"US","type":"company","lineage":["https://openalex.org/I4210153776"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wang, Zirui","raw_affiliation_strings":["Apple Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Apple Inc","institution_ids":["https://openalex.org/I4210153776"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":16,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6845999956130981,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6845999956130981,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.10239999741315842,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.04969999939203262,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/llm-based-agents","display_name":"LLM agents","score":0.722000002861023},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.7020000219345093},{"id":"https://openalex.org/keywords/benchmark","display_name":"benchmark","score":0.5350000262260437},{"id":"https://openalex.org/keywords/agent-capabilities","display_name":"agent capabilities","score":0.6029999852180481},{"id":"https://openalex.org/keywords/tool-use","display_name":"tool use","score":0.8159999847412109},{"id":"https://openalex.org/keywords/programming-contests","display_name":"programming competition","score":0.8259999752044678},{"id":"https://openalex.org/keywords/mathematical-problem-solving","display_name":"mathematical problem solving","score":0.8399999737739563},{"id":"https://openalex.org/keywords/reasoning","display_name":"reasoning","score":0.6019999980926514},{"id":"https://openalex.org/keywords/planning","display_name":"planning","score":0.9639999866485596},{"id":"https://openalex.org/keywords/self-correction","display_name":"self-correction","score":0.8510000109672546},{"id":"https://openalex.org/keywords/offline-evaluation","display_name":"offline evaluation","score":0.48899999260902405},{"id":"https://openalex.org/keywords/interpretability","display_name":"interpretability","score":0.6850000023841858}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.809464156627655},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4788900911808014},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.3403310179710388},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.33466780185699463},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.32723987102508545},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.18878784775733948},{"id":"https://openalex.org/C58640448","wikidata":"https://www.wikidata.org/wiki/Q42515","display_name":"Cartography","level":1,"score":0.09125134348869324}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2407.18961","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2407.18961","pdf_url":"https://arxiv.org/pdf/2407.18961","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2407.18961","doi":"https://doi.org/10.48550/arxiv.2407.18961","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2407.18961","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2407.18961","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2407.18961","pdf_url":"https://arxiv.org/pdf/2407.18961","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4401201041.pdf","grobid_xml":"https://content.openalex.org/works/W4401201041.grobid-xml"},"referenced_works_count":18,"referenced_works":["https://openalex.org/W2912007050","https://openalex.org/W3083410900","https://openalex.org/W4226287673","https://openalex.org/W4319453300","https://openalex.org/W4327810158","https://openalex.org/W4366999541","https://openalex.org/W4385327683","https://openalex.org/W4385682136","https://openalex.org/W4385965642","https://openalex.org/W4386081870","https://openalex.org/W4386528753","https://openalex.org/W4392679398","https://openalex.org/W4402667024","https://openalex.org/W4402716477","https://openalex.org/W4415797149","https://openalex.org/W7133220405","https://openalex.org/W7133222210","https://openalex.org/W7133237354"],"related_works":["https://openalex.org/W7159956031","https://openalex.org/W4385682136","https://openalex.org/W7163707883","https://openalex.org/W7081566077","https://openalex.org/W7164744345","https://openalex.org/W7162448080","https://openalex.org/W4392222647","https://openalex.org/W4394710846","https://openalex.org/W4404782996","https://openalex.org/W7169543074"],"abstract_inverted_index":{"Recent":[0],"advances":[1],"in":[2,76],"large":[3],"language":[4],"models":[5,106,169],"(LLMs)":[6],"have":[7],"increased":[8],"the":[9,38,85,98,158,185,195],"demand":[10],"for":[11,100,156],"comprehensive":[12,93,154],"benchmarks":[13],"to":[14,36,52],"evaluate":[15],"their":[16,198],"capabilities":[17,186],"as":[18],"human-like":[19],"agents.":[20,164],"Existing":[21],"benchmarks,":[22],"while":[23],"useful,":[24],"often":[25],"focus":[26],"on":[27,170,184],"specific":[28],"application":[29],"scenarios,":[30],"emphasizing":[31],"task":[32],"completion":[33],"but":[34,192],"failing":[35],"dissect":[37],"underlying":[39],"skills":[40],"that":[41,96],"drive":[42],"these":[43,62,81],"outcomes.":[44],"This":[45],"lack":[46],"of":[47,69,141,162,189,197,204],"granularity":[48],"makes":[49],"it":[50],"difficult":[51],"deeply":[53],"discern":[54],"where":[55],"failures":[56],"stem":[57],"from.":[58],"Additionally,":[59],"setting":[60],"up":[61],"environments":[63],"requires":[64],"considerable":[65],"effort,":[66],"and":[67,71,119,125,127,136,160,175,187,201],"issues":[68],"unreliability":[70],"reproducibility":[72],"sometimes":[73],"arise,":[74],"especially":[75],"interactive":[77],"tasks.":[78],"To":[79],"address":[80],"limitations,":[82],"we":[83,172],"introduce":[84],"Massive":[86],"Multitask":[87],"Agent":[88],"Understanding":[89],"(MMAU)":[90],"benchmark,":[91],"featuring":[92],"offline":[94],"tasks":[95,145],"eliminate":[97],"need":[99],"complex":[101],"environment":[102],"setups.":[103],"It":[104],"evaluates":[105],"across":[107],"five":[108,129],"domains,":[109],"including":[110],"Tool-use,":[111],"Directed":[112],"Acyclic":[113],"Graph":[114],"(DAG)":[115],"QA,":[116],"Data":[117],"Science":[118],"Machine":[120],"Learning":[121],"coding,":[122],"Contest-level":[123],"programming":[124],"Mathematics,":[126],"covers":[128],"essential":[130],"capabilities:":[131],"Understanding,":[132],"Reasoning,":[133],"Planning,":[134],"Problem-solving,":[135],"Self-correction.":[137],"With":[138],"a":[139,153],"total":[140],"20":[142],"meticulously":[143],"designed":[144],"encompassing":[146],"over":[147],"3K":[148],"distinct":[149],"prompts,":[150],"MMAU":[151,179,205],"provides":[152],"framework":[155],"evaluating":[157],"strengths":[159],"limitations":[161,188],"LLM":[163,190],"By":[165],"testing":[166],"18":[167],"representative":[168],"MMAU,":[171],"provide":[173],"deep":[174],"insightful":[176],"analyses.":[177],"Ultimately,":[178],"not":[180],"only":[181],"sheds":[182],"light":[183],"agents":[191],"also":[193],"enhances":[194],"interpretability":[196],"performance.":[199],"Datasets":[200],"evaluation":[202],"scripts":[203],"are":[206],"released":[207],"at":[208],"https://github.com/apple/axlearn/tree/main/docs/research/mmau.":[209]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":12}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2024-08-01T00:00:00"}