{"id":"https://openalex.org/W7153817334","doi":"https://doi.org/10.1145/3772318.3791178","title":"SeekUI: Predicting Visual Search Behavior on Graphical User Interfaces with a Reward-Augmented Vision Language Model","display_name":"SeekUI: Predicting Visual Search Behavior on Graphical User Interfaces with a Reward-Augmented Vision Language Model","publication_year":2026,"publication_date":"2026-04-13","ids":{"openalex":"https://openalex.org/W7153817334","doi":"https://doi.org/10.1145/3772318.3791178"},"language":null,"primary_location":{"id":"doi:10.1145/3772318.3791178","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3772318.3791178","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3772318.3791178","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Zixin Guo","orcid":"https://orcid.org/0000-0002-7088-2331"},"institutions":[{"id":"https://openalex.org/I9927081","display_name":"Aalto University","ror":"https://ror.org/020hwjq30","country_code":"FI","type":"education","lineage":["https://openalex.org/I9927081"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Zixin Guo","raw_affiliation_strings":["Aalto University, Espoo, Finland"],"raw_orcid":"https://orcid.org/0000-0002-7088-2331","affiliations":[{"raw_affiliation_string":"Aalto University, Espoo, Finland","institution_ids":["https://openalex.org/I9927081"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060889170","display_name":"Yue Jiang","orcid":"https://orcid.org/0000-0003-0022-6512"},"institutions":[{"id":"https://openalex.org/I223532165","display_name":"University of Utah","ror":"https://ror.org/03r0ha626","country_code":"US","type":"education","lineage":["https://openalex.org/I223532165"]},{"id":"https://openalex.org/I9927081","display_name":"Aalto University","ror":"https://ror.org/020hwjq30","country_code":"FI","type":"education","lineage":["https://openalex.org/I9927081"]}],"countries":["FI","US"],"is_corresponding":false,"raw_author_name":"Yue Jiang","raw_affiliation_strings":["Department of Information and Communications Engineering, Aalto University, Espoo, Finland and Kahlert School of Computing, University of Utah, Salt Lake City, Utah, USA"],"raw_orcid":"https://orcid.org/0000-0003-0022-6512","affiliations":[{"raw_affiliation_string":"Department of Information and Communications Engineering, Aalto University, Espoo, Finland and Kahlert School of Computing, University of Utah, Salt Lake City, Utah, USA","institution_ids":["https://openalex.org/I223532165","https://openalex.org/I9927081"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017930924","display_name":"Luis A. Leiva","orcid":"https://orcid.org/0000-0002-5011-1847"},"institutions":[{"id":"https://openalex.org/I186903577","display_name":"University of Luxembourg","ror":"https://ror.org/036x5ad56","country_code":"LU","type":"education","lineage":["https://openalex.org/I186903577"]}],"countries":["LU"],"is_corresponding":false,"raw_author_name":"Luis A. Leiva","raw_affiliation_strings":["University of Luxembourg, Esch-sur-Alzette, Luxembourg"],"raw_orcid":"https://orcid.org/0000-0002-5011-1847","affiliations":[{"raw_affiliation_string":"University of Luxembourg, Esch-sur-Alzette, Luxembourg","institution_ids":["https://openalex.org/I186903577"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5123362038","display_name":"Antti Oulasvirta","orcid":null},"institutions":[{"id":"https://openalex.org/I32943570","display_name":"Helsinki Institute for Information Technology","ror":"https://ror.org/05kph4940","country_code":"FI","type":"facility","lineage":["https://openalex.org/I133731052","https://openalex.org/I32943570","https://openalex.org/I9927081"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Antti Oulasvirta","raw_affiliation_strings":["Department of Information and Communications Engineering, Aalto University, Helsinki, Finland and ELLIS Institute Finland, Helsinki, Finland"],"raw_orcid":"https://orcid.org/0000-0002-2498-7837","affiliations":[{"raw_affiliation_string":"Department of Information and Communications Engineering, Aalto University, Helsinki, Finland and ELLIS Institute Finland, Helsinki, Finland","institution_ids":["https://openalex.org/I32943570"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"23"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.19419999420642853,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.19419999420642853,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10799","display_name":"Data Visualization and Analytics","score":0.15289999544620514,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11707","display_name":"Gaze Tracking and Assistive Technology","score":0.11180000007152557,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/visual-language","display_name":"Visual language","score":0.5231000185012817},{"id":"https://openalex.org/keywords/user-interface","display_name":"User interface","score":0.4964999854564667},{"id":"https://openalex.org/keywords/graphical-user-interface","display_name":"Graphical user interface","score":0.4327000081539154},{"id":"https://openalex.org/keywords/visual-search","display_name":"Visual search","score":0.43230000138282776},{"id":"https://openalex.org/keywords/visualization","display_name":"Visualization","score":0.3598000109195709},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.35580000281333923},{"id":"https://openalex.org/keywords/user-modeling","display_name":"User modeling","score":0.33880001306533813},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.31349998712539673}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.733299970626831},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.5609999895095825},{"id":"https://openalex.org/C2780878386","wikidata":"https://www.wikidata.org/wiki/Q1659648","display_name":"Visual language","level":2,"score":0.5231000185012817},{"id":"https://openalex.org/C89505385","wikidata":"https://www.wikidata.org/wiki/Q47146","display_name":"User interface","level":2,"score":0.4964999854564667},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.46799999475479126},{"id":"https://openalex.org/C37789001","wikidata":"https://www.wikidata.org/wiki/Q782543","display_name":"Graphical user interface","level":2,"score":0.4327000081539154},{"id":"https://openalex.org/C158495155","wikidata":"https://www.wikidata.org/wiki/Q2369151","display_name":"Visual search","level":2,"score":0.43230000138282776},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.3598000109195709},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.35899999737739563},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.35899999737739563},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.35580000281333923},{"id":"https://openalex.org/C67712803","wikidata":"https://www.wikidata.org/wiki/Q7901853","display_name":"User modeling","level":3,"score":0.33880001306533813},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.31349998712539673},{"id":"https://openalex.org/C56461940","wikidata":"https://www.wikidata.org/wiki/Q970687","display_name":"Eye tracking","level":2,"score":0.2989000082015991},{"id":"https://openalex.org/C113843644","wikidata":"https://www.wikidata.org/wiki/Q901882","display_name":"Interface (matter)","level":4,"score":0.26969999074935913},{"id":"https://openalex.org/C155846161","wikidata":"https://www.wikidata.org/wiki/Q1143367","display_name":"Graphical model","level":2,"score":0.2687000036239624},{"id":"https://openalex.org/C2781238097","wikidata":"https://www.wikidata.org/wiki/Q175026","display_name":"Object (grammar)","level":2,"score":0.2680000066757202},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.26759999990463257},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.2662999927997589},{"id":"https://openalex.org/C174252522","wikidata":"https://www.wikidata.org/wiki/Q3816772","display_name":"Natural language user interface","level":3,"score":0.2623000144958496},{"id":"https://openalex.org/C187482481","wikidata":"https://www.wikidata.org/wiki/Q1851756","display_name":"Natural user interface","level":4,"score":0.2621999979019165},{"id":"https://openalex.org/C5339829","wikidata":"https://www.wikidata.org/wiki/Q1425977","display_name":"Machine vision","level":2,"score":0.26100000739097595},{"id":"https://openalex.org/C2984927552","wikidata":"https://www.wikidata.org/wiki/Q845734","display_name":"Graphical display","level":2,"score":0.2549999952316284},{"id":"https://openalex.org/C2778751112","wikidata":"https://www.wikidata.org/wiki/Q835016","display_name":"Window (computing)","level":2,"score":0.25450000166893005},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.2502000033855438}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3772318.3791178","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3772318.3791178","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3772318.3791178","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3772318.3791178","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":88,"referenced_works":["https://openalex.org/W1510835000","https://openalex.org/W1883067458","https://openalex.org/W1964380033","https://openalex.org/W1965729921","https://openalex.org/W1968595779","https://openalex.org/W1975802433","https://openalex.org/W1977965306","https://openalex.org/W1983054435","https://openalex.org/W1985270055","https://openalex.org/W1986533012","https://openalex.org/W1991034237","https://openalex.org/W2015831141","https://openalex.org/W2026558133","https://openalex.org/W2030031014","https://openalex.org/W2032904918","https://openalex.org/W2055111849","https://openalex.org/W2055941205","https://openalex.org/W2067093315","https://openalex.org/W2069192723","https://openalex.org/W2074231493","https://openalex.org/W2076191571","https://openalex.org/W2081216294","https://openalex.org/W2081456979","https://openalex.org/W2081937983","https://openalex.org/W2082627290","https://openalex.org/W2085781384","https://openalex.org/W2088574598","https://openalex.org/W2089623033","https://openalex.org/W2096304920","https://openalex.org/W2109168899","https://openalex.org/W2117160827","https://openalex.org/W2117242469","https://openalex.org/W2118109377","https://openalex.org/W2128272608","https://openalex.org/W2136243608","https://openalex.org/W2143668817","https://openalex.org/W2149095485","https://openalex.org/W2167697220","https://openalex.org/W2169949291","https://openalex.org/W2302086703","https://openalex.org/W2316542548","https://openalex.org/W2400341429","https://openalex.org/W2577731433","https://openalex.org/W2611632447","https://openalex.org/W2799224386","https://openalex.org/W2808707360","https://openalex.org/W2884414611","https://openalex.org/W2891817692","https://openalex.org/W2895004790","https://openalex.org/W2912614787","https://openalex.org/W2914885528","https://openalex.org/W2930891713","https://openalex.org/W2941272892","https://openalex.org/W2964279441","https://openalex.org/W2972101714","https://openalex.org/W2993760950","https://openalex.org/W3004178645","https://openalex.org/W3022455436","https://openalex.org/W3032751033","https://openalex.org/W3034964972","https://openalex.org/W3091272181","https://openalex.org/W3128536198","https://openalex.org/W3163756194","https://openalex.org/W3166946369","https://openalex.org/W3176796859","https://openalex.org/W4212911677","https://openalex.org/W4225004727","https://openalex.org/W4225109875","https://openalex.org/W4285467288","https://openalex.org/W4366549548","https://openalex.org/W4366588992","https://openalex.org/W4366590920","https://openalex.org/W4384659783","https://openalex.org/W4386057737","https://openalex.org/W4389296917","https://openalex.org/W4395065140","https://openalex.org/W4396831918","https://openalex.org/W4396833734","https://openalex.org/W4396986748","https://openalex.org/W4399071590","https://openalex.org/W4403333913","https://openalex.org/W4403447917","https://openalex.org/W4404784092","https://openalex.org/W4408080032","https://openalex.org/W4408615011","https://openalex.org/W4409720607","https://openalex.org/W4416034866","https://openalex.org/W7133196460"],"related_works":[],"abstract_inverted_index":{"Visual":[0],"search":[1,27,64,102,141],"is":[2,28],"key":[3,118],"to":[4,98],"understanding":[5],"and":[6,30,35,61,83,106,143,150],"improving":[7],"interaction":[8],"with":[9,51],"graphical":[10],"user":[11],"interfaces":[12],"(GUIs),":[13],"yet":[14],"predicting":[15,139],"scanpaths":[16,77],"on":[17,47,63,104],"real":[18],"GUIs":[19,52,105],"remains":[20],"an":[21],"open":[22],"challenge.":[23],"Unlike":[24],"free-viewing,":[25],"visual":[26,36,44,101,126,140],"goal-driven":[29],"shaped":[31],"by":[32],"both":[33],"linguistic":[34],"features":[37],"of":[38,43,59,96,113,125],"the":[39,57,88,94,129],"GUI.":[40],"State-of-the-art":[41],"models":[42,109],"search,":[45,127],"trained":[46],"natural":[48],"images,":[49],"fail":[50],"because":[53],"they":[54],"cannot":[55],"capture":[56],"effects":[58],"grouping":[60],"semantics":[62],"strategies.":[65],"We":[66],"present":[67],"SeekUI,":[68],"a":[69,80,84,136],"reward-augmented":[70],"Vision":[71],"Language":[72],"Model":[73],"(VLM)":[74],"that":[75],"predicts":[76],"directly":[78],"from":[79],"GUI":[81,148],"screenshot":[82],"text":[85],"cue":[86],"describing":[87],"desired":[89],"target.":[90],"Our":[91],"model":[92],"extends":[93],"capability":[95],"VLMs":[97],"reproduce":[99],"human-like":[100],"behavior":[103,142],"outperforms":[107],"baseline":[108],"across":[110],"different":[111],"types":[112],"GUIs.":[114],"Importantly,":[115],"it":[116],"reproduces":[117],"empirical":[119],"phenomena":[120],"established":[121],"in":[122],"eye-tracking":[123],"studies":[124],"including":[128],"Guess\u2013Scan\u2013Confirm":[130],"strategy.":[131],"In":[132],"sum,":[133],"SeekUI":[134],"provides":[135],"foundation":[137],"for":[138,146],"has":[144],"potential":[145],"informing":[147],"evaluation":[149],"optimization.":[151]},"counts_by_year":[],"updated_date":"2026-07-22T07:51:19.307946","created_date":"2026-04-13T00:00:00"}