{"id":"https://openalex.org/W7138260847","doi":"https://doi.org/10.1609/aaai.v40i35.40175","title":"GUI-Eyes: Tool-Augmented Perception for Visual Grounding in GUI Agents","display_name":"GUI-Eyes: Tool-Augmented Perception for Visual Grounding in GUI Agents","publication_year":2026,"publication_date":"2026-03-14","ids":{"openalex":"https://openalex.org/W7138260847","doi":"https://doi.org/10.1609/aaai.v40i35.40175"},"language":"en","primary_location":{"id":"doi:10.1609/aaai.v40i35.40175","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v40i35.40175","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/40175/44136","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://ojs.aaai.org/index.php/AAAI/article/download/40175/44136","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5129653544","display_name":"Chen Chen","orcid":null},"institutions":[{"id":"https://openalex.org/I4210136246","display_name":"China Telecom (China)","ror":"https://ror.org/03jgnzt20","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210136246"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chen Chen","raw_affiliation_strings":["University of Science and Technology of China\nInstitute of Artificial Intelligence (TeleAI), China Telecom\nShanghai Innovation Institute"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Science and Technology of China\nInstitute of Artificial Intelligence (TeleAI), China Telecom\nShanghai Innovation Institute","institution_ids":["https://openalex.org/I4210136246"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129658649","display_name":"Jiawei Shao","orcid":null},"institutions":[{"id":"https://openalex.org/I4210136246","display_name":"China Telecom (China)","ror":"https://ror.org/03jgnzt20","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210136246"]},{"id":"https://openalex.org/I4387153335","display_name":"China Telecom","ror":"https://ror.org/05p67dv18","country_code":null,"type":"company","lineage":["https://openalex.org/I4387153335"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiawei Shao","raw_affiliation_strings":["Institute of Artificial Intelligence (TeleAI), China Telecom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Artificial Intelligence (TeleAI), China Telecom","institution_ids":["https://openalex.org/I4210136246","https://openalex.org/I4387153335"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129656096","display_name":"Dakuan Lu","orcid":null},"institutions":[{"id":"https://openalex.org/I4210136246","display_name":"China Telecom (China)","ror":"https://ror.org/03jgnzt20","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210136246"]},{"id":"https://openalex.org/I4387153335","display_name":"China Telecom","ror":"https://ror.org/05p67dv18","country_code":null,"type":"company","lineage":["https://openalex.org/I4387153335"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Dakuan Lu","raw_affiliation_strings":["Institute of Artificial Intelligence (TeleAI), China Telecom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Artificial Intelligence (TeleAI), China Telecom","institution_ids":["https://openalex.org/I4210136246","https://openalex.org/I4387153335"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104343859","display_name":"Haoyi Hu","orcid":null},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Haoyi Hu","raw_affiliation_strings":["Shanghai Jiaotong University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai Jiaotong University","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129671866","display_name":"Xiangcheng Liu","orcid":null},"institutions":[{"id":"https://openalex.org/I67001856","display_name":"Shanghai Institute of Technology","ror":"https://ror.org/00fjzqj15","country_code":"CN","type":"education","lineage":["https://openalex.org/I67001856"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiangcheng Liu","raw_affiliation_strings":["University of Science and Technology of China\nShanghai Innovation Institute"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Science and Technology of China\nShanghai Innovation Institute","institution_ids":["https://openalex.org/I67001856"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129673118","display_name":"Hantao Yao","orcid":null},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hantao Yao","raw_affiliation_strings":["University of Science and Technology of China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Science and Technology of China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5129730085","display_name":"Wu Liu","orcid":null},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wu Liu","raw_affiliation_strings":["University of Science and Technology of China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Science and Technology of China","institution_ids":["https://openalex.org/I126520041"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.25578869,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"40","issue":"35","first_page":"29350","last_page":"29358"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.920799970626831,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.920799970626831,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.00860000029206276,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.007600000128149986,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.7182999849319458},{"id":"https://openalex.org/keywords/perception","display_name":"Perception","score":0.679099977016449},{"id":"https://openalex.org/keywords/function","display_name":"Function (biology)","score":0.5436000227928162},{"id":"https://openalex.org/keywords/visual-reasoning","display_name":"Visual reasoning","score":0.5397999882698059},{"id":"https://openalex.org/keywords/visual-perception","display_name":"Visual perception","score":0.43209999799728394},{"id":"https://openalex.org/keywords/visualization","display_name":"Visualization","score":0.40630000829696655},{"id":"https://openalex.org/keywords/active-vision","display_name":"Active vision","score":0.3995000123977661}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7239999771118164},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7182999849319458},{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.679099977016449},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6159999966621399},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.5436000227928162},{"id":"https://openalex.org/C2777508537","wikidata":"https://www.wikidata.org/wiki/Q7936620","display_name":"Visual reasoning","level":2,"score":0.5397999882698059},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.49309998750686646},{"id":"https://openalex.org/C178253425","wikidata":"https://www.wikidata.org/wiki/Q162668","display_name":"Visual perception","level":3,"score":0.43209999799728394},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.40630000829696655},{"id":"https://openalex.org/C193611912","wikidata":"https://www.wikidata.org/wiki/Q4677596","display_name":"Active vision","level":2,"score":0.3995000123977661},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.36809998750686646},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.3228999972343445},{"id":"https://openalex.org/C136389625","wikidata":"https://www.wikidata.org/wiki/Q334384","display_name":"Supervised learning","level":3,"score":0.30730000138282776},{"id":"https://openalex.org/C111370547","wikidata":"https://www.wikidata.org/wiki/Q7451120","display_name":"Sensory cue","level":2,"score":0.29269999265670776},{"id":"https://openalex.org/C77967617","wikidata":"https://www.wikidata.org/wiki/Q4677561","display_name":"Active learning (machine learning)","level":2,"score":0.2793999910354614},{"id":"https://openalex.org/C2776010242","wikidata":"https://www.wikidata.org/wiki/Q4677575","display_name":"Active perception","level":3,"score":0.2784999907016754},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.26260000467300415},{"id":"https://openalex.org/C155911833","wikidata":"https://www.wikidata.org/wiki/Q3817354","display_name":"Spatial intelligence","level":2,"score":0.2567000091075897}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1609/aaai.v40i35.40175","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v40i35.40175","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/40175/44136","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},{"id":"pmh:oai:ojs.aaai.org:article/40175","is_oa":false,"landing_page_url":"https://ojs.aaai.org/index.php/AAAI/article/view/40175","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"2159-5399","raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":{"id":"doi:10.1609/aaai.v40i35.40175","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v40i35.40175","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/40175/44136","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16","score":0.7638993859291077}],"awards":[{"id":"https://openalex.org/G2321388553","display_name":null,"funder_award_id":"U24A20329","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320335777","display_name":"National Key Research and Development Program of China","ror":null}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7138260847.pdf","grobid_xml":"https://content.openalex.org/works/W7138260847.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Recent":[0],"advances":[1],"in":[2,13,54,144],"vision-language":[3],"models":[4],"(VLMs)":[5],"and":[6,26,37,72,105,131,138,165,180,189],"reinforcement":[7,47],"learning":[8,48],"(RL)":[9],"have":[10],"driven":[11],"progress":[12],"GUI":[14,55,145,191],"automation.":[15],"However,":[16],"most":[17],"existing":[18],"methods":[19],"rely":[20],"on":[21,69],"static,":[22],"one-shot":[23],"visual":[24,52,76],"inputs":[25],"passive":[27],"perception,":[28,174],"lacking":[29],"the":[30,41,62,100,140,148],"ability":[31],"to":[32,39,65,74,123,134],"adaptively":[33],"determine":[34],"when,":[35],"whether,":[36],"how":[38,73],"observe":[40],"interface.":[42],"We":[43],"present":[44],"GUI-Eyes,":[45],"a":[46,84,94,110,117],"framework":[49],"for":[50,186],"active":[51,173],"perception":[53,96],"tasks.":[56],"To":[57,88],"acquire":[58],"more":[59],"informative":[60],"observations,":[61],"agent":[63],"learns":[64],"make":[66],"strategic":[67],"decisions":[68],"both":[70,128,163],"whether":[71],"invoke":[75],"tools,":[77],"such":[78],"as":[79],"cropping":[80],"or":[81],"zooming,":[82],"within":[83],"two-stage":[85],"reasoning":[86,179],"process.":[87],"support":[89],"this":[90],"behavior,":[91],"we":[92,115],"introduce":[93],"progressive":[95],"strategy":[97],"that":[98,171],"decomposes":[99],"decision-making":[101],"into":[102],"coarse":[103],"exploration":[104],"fine-grained":[106,181],"grounding,":[107],"coordinated":[108],"by":[109,176],"two-level":[111],"policy.":[112],"In":[113],"addition,":[114],"design":[116],"spatially":[118],"continuous":[119],"reward":[120,141,182],"function":[121],"tailored":[122],"tool":[124],"usage,":[125],"which":[126],"integrates":[127],"location":[129],"proximity":[130],"region":[132],"overlap":[133],"provide":[135],"dense":[136],"supervision":[137],"alleviate":[139],"sparsity":[142],"common":[143],"environments.":[146],"On":[147],"ScreenSpot-Pro":[149],"benchmark,":[150],"GUI-Eyes-3B":[151],"achieves":[152],"44.8%":[153],"grounding":[154],"accuracy":[155],"using":[156],"only":[157],"3k":[158],"labeled":[159],"samples,":[160],"significantly":[161],"outperforming":[162],"supervised":[164],"RL-based":[166],"baselines.":[167],"These":[168],"results":[169],"highlight":[170],"tool-aware":[172],"enabled":[175],"staged":[177],"policy":[178],"feedback,":[183],"is":[184],"critical":[185],"building":[187],"robust":[188],"data-efficient":[190],"agents.":[192]},"counts_by_year":[],"updated_date":"2026-08-26T07:47:46.906454","created_date":"2026-03-18T00:00:00"}