{"id":"https://openalex.org/W7165672602","doi":"https://doi.org/10.48550/arxiv.2606.22557","title":"MacAgentBench: Benchmarking AI Agents on Real-World macOS Desktop","display_name":"MacAgentBench: Benchmarking AI Agents on Real-World macOS Desktop","publication_year":2026,"publication_date":"2026-06-21","ids":{"openalex":"https://openalex.org/W7165672602","doi":"https://doi.org/10.48550/arxiv.2606.22557"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.22557","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.22557","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.22557","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5069101640","display_name":"Y Fu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fu, Yikun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139162585","display_name":"Bowen Fu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fu, Bowen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139198466","display_name":"Zhenyu Wu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Zhenyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139152181","display_name":"Shuang Cheng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cheng, Shuang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139136546","display_name":"Xiaowei Sun","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sun, Xiaowei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5122931632","display_name":"B Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Bowen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139128333","display_name":"Zehao Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Zehao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136047676","display_name":"Yibo Zhao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao, Yibo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139214519","display_name":"Zichen Ding","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ding, Zichen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5049003506","display_name":"Zhoumianze Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Zhoumianze","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100374850","display_name":"Shijie Wang","orcid":"https://orcid.org/0000-0002-7990-4883"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Shijie","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048953650","display_name":"Biqing Qi","orcid":"https://orcid.org/0000-0002-4072-0577"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qi, Biqing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5139143813","display_name":"Bowen Zhou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhou, Bowen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12607","display_name":"Personal Information Management and User Behavior","score":0.20600000023841858,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T12607","display_name":"Personal Information Management and User Behavior","score":0.20600000023841858,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10639","display_name":"Advanced Software Engineering Methodologies","score":0.14730000495910645,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11424","display_name":"Security and Verification in Computing","score":0.07760000228881836,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.8396999835968018},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.7817999720573425},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.5388000011444092},{"id":"https://openalex.org/keywords/source-code","display_name":"Source code","score":0.42089998722076416},{"id":"https://openalex.org/keywords/binary-number","display_name":"Binary number","score":0.3598000109195709},{"id":"https://openalex.org/keywords/os-x","display_name":"OS X","score":0.3043999969959259}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.8396999835968018},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7817999720573425},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7728999853134155},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5388000011444092},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.42649999260902405},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.42089998722076416},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.3763999938964844},{"id":"https://openalex.org/C48372109","wikidata":"https://www.wikidata.org/wiki/Q3913","display_name":"Binary number","level":2,"score":0.3598000109195709},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.31150001287460327},{"id":"https://openalex.org/C515153823","wikidata":"https://www.wikidata.org/wiki/Q14116","display_name":"OS X","level":3,"score":0.3043999969959259},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.302700012922287},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.29409998655319214},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2935999929904938},{"id":"https://openalex.org/C63435697","wikidata":"https://www.wikidata.org/wiki/Q864135","display_name":"Binary code","level":3,"score":0.289000004529953},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2800000011920929},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.2718000113964081},{"id":"https://openalex.org/C76518257","wikidata":"https://www.wikidata.org/wiki/Q271680","display_name":"Software framework","level":5,"score":0.25699999928474426}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.22557","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.22557","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.22557","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.22557","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Computer":[0],"use":[1],"agents":[2,35],"(CUAs)":[3],"have":[4],"advanced":[5],"rapidly":[6],"in":[7,154],"desktop":[8],"automation,":[9],"and":[10,39,59,87,96,111,159],"a":[11,45,70],"growing":[12],"number":[13],"of":[14],"users":[15],"deploy":[16],"CUAs":[17,58],"such":[18],"as":[19],"OpenClaw":[20],"on":[21,41,63,122],"Mac":[22],"Mini":[23],"for":[24,32,104],"always-on":[25],"automation.":[26],"However,":[27],"existing":[28],"benchmarks,":[29],"including":[30],"those":[31],"macOS,":[33],"evaluate":[34],"without":[36],"framework":[37,53,140],"augmentation":[38],"rely":[40],"binary":[42],"evaluation.":[43],"As":[44],"result,":[46],"they":[47],"fail":[48],"to":[49],"capture":[50],"both":[51,85],"the":[52,60,116,134],"capabilities":[54],"leveraged":[55],"by":[56,133,139],"modern":[57],"partial":[61],"progress":[62],"long-horizon,":[64],"multi-application":[65,105],"tasks.":[66,106],"We":[67],"present":[68],"MacAgentBench,":[69],"comprehensive":[71],"macOS":[72],"agent":[73],"benchmark":[74,91],"comprising":[75],"676":[76],"tasks":[77],"across":[78,108],"25":[79],"applications,":[80],"with":[81,101,148],"nearly":[82],"60%":[83],"involving":[84],"GUI":[86],"CLI":[88],"interaction.":[89],"The":[90],"adopts":[92],"deterministic":[93],"rule-based":[94],"evaluation":[95],"introduces":[97],"fine-grained":[98],"multi-checkpoint":[99],"scoring":[100],"capability":[102],"annotations":[103],"Experiments":[107],"three":[109],"frameworks":[110],"16":[112],"models":[113,147],"show":[114],"that":[115,146],"best":[117],"configuration,":[118],"Claude":[119],"Opus":[120],"4.6":[121],"OpenClaw,":[123],"attains":[124],"73.7%":[125],"Pass@1,":[126],"while":[127],"this":[128],"advantage":[129],"is":[130],"primarily":[131],"driven":[132],"skill":[135],"library":[136],"rather":[137],"than":[138],"design.":[141],"Fine-grained":[142],"metrics":[143],"further":[144],"reveal":[145],"similar":[149],"Pass@1":[150],"can":[151],"differ":[152],"substantially":[153],"sub-goal":[155],"completion.":[156],"Our":[157],"code":[158],"data":[160],"are":[161],"publicly":[162],"available":[163],"at":[164],"https://github.com/JetAstra/MacAgentBench.":[165]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-24T00:00:00"}