{"id":"https://openalex.org/W7161021477","doi":"https://doi.org/10.48550/arxiv.2605.12501","title":"Covering Human Action Space for Computer Use: Data Synthesis and Benchmark","display_name":"Covering Human Action Space for Computer Use: Data Synthesis and Benchmark","publication_year":2026,"publication_date":"2026-05-12","ids":{"openalex":"https://openalex.org/W7161021477","doi":"https://doi.org/10.48550/arxiv.2605.12501"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.12501","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.12501","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.12501","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5136044687","display_name":"Miaosen Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Miaosen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136053928","display_name":"Xiaohan Zhao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao, Xiaohan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136066084","display_name":"Zhihong Tan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tan, Zhihong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136040211","display_name":"Zhou Huoshen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huoshen, Zhou","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136070120","display_name":"Yijia Fan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fan, Yijia","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136057834","display_name":"Yifan Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Yifan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102742709","display_name":"Kai Qiu","orcid":"https://orcid.org/0000-0003-2071-4425"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qiu, Kai","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136069995","display_name":"Bei Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Bei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136076222","display_name":"Justin Wagle","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wagle, Justin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037487522","display_name":"Chenzhong Yin","orcid":"https://orcid.org/0000-0001-6411-7441"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yin, Chenzhong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136087132","display_name":"Mingxi Cheng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cheng, Mingxi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136073690","display_name":"Ji Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Ji","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000696653","display_name":"Qi Dai","orcid":"https://orcid.org/0000-0002-9920-784X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dai, Qi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5124872859","display_name":"Chong Luo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Luo, Chong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136078686","display_name":"Xu Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Xu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136015759","display_name":"Xin Geng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Geng, Xin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5136068753","display_name":"Baining Guo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Guo, Baining","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13197","display_name":"Spreadsheets and End-User Computing","score":0.2264000028371811,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13197","display_name":"Spreadsheets and End-User Computing","score":0.2264000028371811,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10789","display_name":"Interactive and Immersive Displays","score":0.14000000059604645,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10709","display_name":"Social Robot Interaction and HRI","score":0.08789999783039093,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.7545999884605408},{"id":"https://openalex.org/keywords/variety","display_name":"Variety (cybernetics)","score":0.5837000012397766},{"id":"https://openalex.org/keywords/matching","display_name":"Matching (statistics)","score":0.579200029373169},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5493000149726868},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.492000013589859},{"id":"https://openalex.org/keywords/limiting","display_name":"Limiting","score":0.49140000343322754},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.453000009059906},{"id":"https://openalex.org/keywords/action","display_name":"Action (physics)","score":0.4481000006198883},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.3499999940395355}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7633000016212463},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7545999884605408},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.5837000012397766},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.579200029373169},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5493000149726868},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.492000013589859},{"id":"https://openalex.org/C188198153","wikidata":"https://www.wikidata.org/wiki/Q1613840","display_name":"Limiting","level":2,"score":0.49140000343322754},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.453000009059906},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.4481000006198883},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4431999921798706},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.39160001277923584},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3788999915122986},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.3499999940395355},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.34619998931884766},{"id":"https://openalex.org/C2778572836","wikidata":"https://www.wikidata.org/wiki/Q380933","display_name":"Space (punctuation)","level":2,"score":0.34610000252723694},{"id":"https://openalex.org/C155846161","wikidata":"https://www.wikidata.org/wiki/Q1143367","display_name":"Graphical model","level":2,"score":0.33329999446868896},{"id":"https://openalex.org/C138958017","wikidata":"https://www.wikidata.org/wiki/Q190087","display_name":"Data type","level":2,"score":0.32359999418258667},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.31540000438690186},{"id":"https://openalex.org/C109747225","wikidata":"https://www.wikidata.org/wiki/Q815758","display_name":"Scarcity","level":2,"score":0.3034000098705292},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.290800005197525},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.2879999876022339},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.27709999680519104},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.271699994802475},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.26409998536109924},{"id":"https://openalex.org/C68859911","wikidata":"https://www.wikidata.org/wiki/Q1503724","display_name":"Pattern matching","level":2,"score":0.26190000772476196},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.2605000138282776},{"id":"https://openalex.org/C200288055","wikidata":"https://www.wikidata.org/wiki/Q2621792","display_name":"Element (criminal law)","level":2,"score":0.2599000036716461},{"id":"https://openalex.org/C2988382989","wikidata":"https://www.wikidata.org/wiki/Q370685","display_name":"Data space","level":2,"score":0.2583000063896179}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.12501","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.12501","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.12501","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.12501","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/1","score":0.5842124819755554,"display_name":"No poverty"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Computer-use":[0],"agents":[1],"(CUAs)":[2],"automate":[3],"on-screen":[4],"work,":[5],"as":[6,100,102],"illustrated":[7],"by":[8],"GPT-5.4":[9],"and":[10,47,97,143,148,154,179],"Claude.":[11],"Yet":[12],"their":[13],"reliability":[14],"on":[15,87,125,159],"complex,":[16],"low-frequency":[17],"interactions":[18,49,89],"is":[19],"still":[20],"poor,":[21],"limiting":[22],"user":[23],"trust.":[24],"Our":[25],"analysis":[26],"of":[27,45,55,68,105,115],"failure":[28],"cases":[29],"from":[30,65],"advanced":[31],"models":[32,166,180],"suggests":[33],"a":[34,41,52,79,103,112,131],"long-tail":[35],"pattern":[36],"in":[37],"GUI":[38,126],"operations,":[39],"where":[40],"relatively":[42],"small":[43],"fraction":[44],"complex":[46,71,88],"diverse":[48],"accounts":[50],"for":[51,70,83,139],"disproportionate":[53],"share":[54],"task":[56],"failures.":[57],"We":[58,128,172],"hypothesize":[59],"that":[60,122],"this":[61,75,160],"issue":[62],"largely":[63],"stems":[64],"the":[66],"scarcity":[67],"data":[69],"interactions.":[72],"To":[73],"address":[74],"problem,":[76],"we":[77],"propose":[78],"new":[80],"benchmark":[81],"CUActSpot":[82],"evaluating":[84],"models'":[85],"capabilities":[86],"across":[90],"five":[91],"modalities:":[92],"GUI,":[93],"text,":[94],"table,":[95],"canvas,":[96],"natural":[98],"image,":[99],"well":[101],"variety":[104],"actions":[106],"(click,":[107],"drag,":[108],"draw,":[109],"etc.),":[110],"covering":[111],"broader":[113],"range":[114],"interaction":[116],"types":[117],"than":[118,169],"prior":[119],"click-centric":[120],"benchmarks":[121],"focus":[123],"mainly":[124],"widgets.":[127],"also":[129],"design":[130],"renderer-based":[132],"data-synthesis":[133],"pipeline:":[134],"scenes":[135],"are":[136,146],"automatically":[137],"generated":[138],"each":[140],"modality,":[141],"screenshots":[142],"element":[144],"coordinates":[145],"recorded,":[147],"an":[149],"LLM":[150],"produces":[151],"matching":[152],"instructions":[153],"action":[155],"traces.":[156],"After":[157],"training":[158],"corpus,":[161],"our":[162,175],"Phi-Ground-Any-4B":[163],"outperforms":[164],"open-source":[165],"with":[167],"fewer":[168],"32B":[170],"parameters.":[171],"will":[173],"release":[174],"benchmark,":[176],"data,":[177],"code,":[178],"at":[181],"https://github.com/microsoft/Phi-Ground.git":[182]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-14T00:00:00"}