{"id":"https://openalex.org/W7158686769","doi":"https://doi.org/10.48550/arxiv.2604.26020","title":"Training Computer Use Agents to Assess the Usability of Graphical User Interfaces","display_name":"Training Computer Use Agents to Assess the Usability of Graphical User Interfaces","publication_year":2026,"publication_date":"2026-04-28","ids":{"openalex":"https://openalex.org/W7158686769","doi":"https://doi.org/10.48550/arxiv.2604.26020"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.26020","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.26020","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.26020","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5134875962","display_name":"Alice Gao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gao, Alice","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134249417","display_name":"Weixi Tong","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tong, Weixi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134898751","display_name":"Rishab Vempati","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Vempati, Rishab","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076852435","display_name":"Katharina Reinecke","orcid":"https://orcid.org/0000-0001-7897-9325"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Reinecke, Katharina","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134882884","display_name":"R. Benjamin Shapiro","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shapiro, R. Benjamin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134899624","display_name":"Tianyi Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Tianyi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5101449235","display_name":"Jason Wu","orcid":"https://orcid.org/0000-0003-2073-3562"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Jason","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10470","display_name":"Usability and User Interface Design","score":0.7892000079154968,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10470","display_name":"Usability and User Interface Design","score":0.7892000079154968,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12607","display_name":"Personal Information Management and User Behavior","score":0.07249999791383743,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10789","display_name":"Interactive and Immersive Displays","score":0.06650000065565109,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/usability","display_name":"Usability","score":0.9302999973297119},{"id":"https://openalex.org/keywords/cognitive-walkthrough","display_name":"Cognitive walkthrough","score":0.652899980545044},{"id":"https://openalex.org/keywords/usability-lab","display_name":"Usability lab","score":0.6247000098228455},{"id":"https://openalex.org/keywords/usability-inspection","display_name":"Usability inspection","score":0.6171000003814697},{"id":"https://openalex.org/keywords/usability-engineering","display_name":"Usability engineering","score":0.5963000059127808},{"id":"https://openalex.org/keywords/heuristic-evaluation","display_name":"Heuristic evaluation","score":0.542900025844574},{"id":"https://openalex.org/keywords/usability-goals","display_name":"Usability goals","score":0.499099999666214},{"id":"https://openalex.org/keywords/web-usability","display_name":"Web usability","score":0.4738999903202057}],"concepts":[{"id":"https://openalex.org/C170130773","wikidata":"https://www.wikidata.org/wiki/Q216378","display_name":"Usability","level":2,"score":0.9302999973297119},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.8057000041007996},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7718999981880188},{"id":"https://openalex.org/C87105883","wikidata":"https://www.wikidata.org/wiki/Q1107002","display_name":"Cognitive walkthrough","level":4,"score":0.652899980545044},{"id":"https://openalex.org/C11324603","wikidata":"https://www.wikidata.org/wiki/Q2502322","display_name":"Usability lab","level":4,"score":0.6247000098228455},{"id":"https://openalex.org/C23456302","wikidata":"https://www.wikidata.org/wiki/Q7901668","display_name":"Usability inspection","level":4,"score":0.6171000003814697},{"id":"https://openalex.org/C100302975","wikidata":"https://www.wikidata.org/wiki/Q1642623","display_name":"Usability engineering","level":3,"score":0.5963000059127808},{"id":"https://openalex.org/C3255780","wikidata":"https://www.wikidata.org/wiki/Q1616517","display_name":"Heuristic evaluation","level":3,"score":0.542900025844574},{"id":"https://openalex.org/C62993174","wikidata":"https://www.wikidata.org/wiki/Q2928808","display_name":"Usability goals","level":4,"score":0.499099999666214},{"id":"https://openalex.org/C4237393","wikidata":"https://www.wikidata.org/wiki/Q1636686","display_name":"Web usability","level":3,"score":0.4738999903202057},{"id":"https://openalex.org/C89505385","wikidata":"https://www.wikidata.org/wiki/Q47146","display_name":"User interface","level":2,"score":0.47099998593330383},{"id":"https://openalex.org/C37789001","wikidata":"https://www.wikidata.org/wiki/Q782543","display_name":"Graphical user interface","level":2,"score":0.3474000096321106},{"id":"https://openalex.org/C188688815","wikidata":"https://www.wikidata.org/wiki/Q7205541","display_name":"Pluralistic walkthrough","level":3,"score":0.3343999981880188},{"id":"https://openalex.org/C67712803","wikidata":"https://www.wikidata.org/wiki/Q7901853","display_name":"User modeling","level":3,"score":0.3111000061035156},{"id":"https://openalex.org/C71151206","wikidata":"https://www.wikidata.org/wiki/Q5156686","display_name":"Component-based usability testing","level":5,"score":0.30000001192092896},{"id":"https://openalex.org/C201025465","wikidata":"https://www.wikidata.org/wiki/Q11248500","display_name":"User experience design","level":2,"score":0.2824999988079071},{"id":"https://openalex.org/C63880386","wikidata":"https://www.wikidata.org/wiki/Q5157592","display_name":"Computer user satisfaction","level":4,"score":0.26750001311302185},{"id":"https://openalex.org/C139225968","wikidata":"https://www.wikidata.org/wiki/Q17146354","display_name":"System usability scale","level":4,"score":0.2630000114440918},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.2549999952316284}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.26020","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.26020","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.26020","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.26020","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Usability":[0],"testing":[1],"with":[2,111,125],"experts":[3],"and":[4,12,26,37,46,96,128,142,149],"potential":[5],"users":[6],"can":[7,42],"assess":[8,81],"the":[9],"effectiveness,":[10],"efficiency,":[11],"user":[13,17,44,121],"satisfaction":[14],"of":[15,75,118,146],"graphical":[16],"interfaces":[18,122],"(GUIs)":[19],"but":[20,48],"doing":[21],"so":[22],"remains":[23],"a":[24,65,72,99,106,115,159],"costly":[25],"time-intensive":[27],"process.":[28],"Prior":[29],"work":[30,155],"has":[31],"used":[32],"computer":[33,107],"use":[34,108],"agents":[35,40,52],"(CUAs)":[36],"other":[38],"generative":[39],"that":[41,51,70,133],"simulate":[43],"interactions":[45],"preference,":[47],"we":[49,63],"show":[50,132],"still":[53],"struggle":[54],"to":[55,77,80,157],"provide":[56],"accurate":[57,139],"usability":[58,76,83,102,126,140,165],"assessments.":[59],"In":[60],"this":[61],"work,":[62],"present":[64],"novel":[66],"machine":[67],"learning":[68],"method":[69],"operationalizes":[71],"computational":[73],"definition":[74],"train":[78,105],"CUAs":[79],"GUI":[82],"by":[84],"i)":[85],"prioritizing":[86],"important":[87],"interaction":[88],"flows,":[89],"ii)":[90],"executing":[91],"them":[92],"through":[93],"human-like":[94],"interactions,":[95],"iii)":[97],"predicting":[98],"learned":[100],"numerical":[101],"score.":[103],"We":[104,131],"agent,":[109],"uxCUA,":[110],"our":[112,154],"algorithm":[113],"on":[114],"large-scale":[116],"dataset":[117],"fully":[119],"interactive":[120],"(UIs)":[123],"paired":[124],"labels":[127],"human":[129],"preferences.":[130],"uxCUA":[134],"outperforms":[135],"larger":[136],"models":[137],"in":[138,167],"assessments":[141],"produces":[143],"realistic":[144],"critiques":[145],"both":[147],"synthetic":[148],"real":[150],"UIs.":[151],"More":[152],"broadly,":[153],"aims":[156],"build":[158],"principled,":[160],"data-driven":[161],"foundation":[162],"for":[163],"automated":[164],"assessment":[166],"HCI.":[168]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-01T00:00:00"}