{"id":"https://openalex.org/W4297220110","doi":"https://doi.org/10.1145/3564702","title":"Describing UI Screenshots in Natural Language","display_name":"Describing UI Screenshots in Natural Language","publication_year":2022,"publication_date":"2022-09-26","ids":{"openalex":"https://openalex.org/W4297220110","doi":"https://doi.org/10.1145/3564702"},"language":"en","primary_location":{"id":"doi:10.1145/3564702","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3564702","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3564702","source":{"id":"https://openalex.org/S2492086750","display_name":"ACM Transactions on Intelligent Systems and Technology","issn_l":"2157-6904","issn":["2157-6904","2157-6912"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Intelligent Systems and Technology","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3564702","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5017930924","display_name":"Luis A. Leiva","orcid":"https://orcid.org/0000-0002-5011-1847"},"institutions":[{"id":"https://openalex.org/I186903577","display_name":"University of Luxembourg","ror":"https://ror.org/036x5ad56","country_code":"LU","type":"education","lineage":["https://openalex.org/I186903577"]}],"countries":["LU"],"is_corresponding":false,"raw_author_name":"Luis A. Leiva","raw_affiliation_strings":["University of Luxembourg, Esch-sur-Alzette, Luxembourg"],"raw_orcid":"https://orcid.org/0000-0002-5011-1847","affiliations":[{"raw_affiliation_string":"University of Luxembourg, Esch-sur-Alzette, Luxembourg","institution_ids":["https://openalex.org/I186903577"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014914239","display_name":"Asutosh Hota","orcid":"https://orcid.org/0000-0003-1453-4452"},"institutions":[{"id":"https://openalex.org/I9927081","display_name":"Aalto University","ror":"https://ror.org/020hwjq30","country_code":"FI","type":"education","lineage":["https://openalex.org/I9927081"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Asutosh Hota","raw_affiliation_strings":["Aalto University, Espoo, Finland"],"raw_orcid":"https://orcid.org/0000-0003-1453-4452","affiliations":[{"raw_affiliation_string":"Aalto University, Espoo, Finland","institution_ids":["https://openalex.org/I9927081"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5003084232","display_name":"Antti Oulasvirta","orcid":"https://orcid.org/0000-0002-2498-7837"},"institutions":[{"id":"https://openalex.org/I9927081","display_name":"Aalto University","ror":"https://ror.org/020hwjq30","country_code":"FI","type":"education","lineage":["https://openalex.org/I9927081"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Antti Oulasvirta","raw_affiliation_strings":["Aalto University, Espoo, Finland"],"raw_orcid":"https://orcid.org/0000-0002-2498-7837","affiliations":[{"raw_affiliation_string":"Aalto University, Espoo, Finland","institution_ids":["https://openalex.org/I9927081"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.2146,"has_fulltext":true,"cited_by_count":15,"citation_normalized_percentile":{"value":0.79035576,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":"14","issue":"1","first_page":"1","last_page":"28"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9896000027656555,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11439","display_name":"Video Analysis and Summarization","score":0.9868000149726868,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.9202311038970947},{"id":"https://openalex.org/keywords/natural-language-user-interface","display_name":"Natural language user interface","score":0.6417635679244995},{"id":"https://openalex.org/keywords/salient","display_name":"Salient","score":0.6404463052749634},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.6383309960365295},{"id":"https://openalex.org/keywords/interface","display_name":"Interface (matter)","score":0.5611523389816284},{"id":"https://openalex.org/keywords/closed-captioning","display_name":"Closed captioning","score":0.5407348871231079},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5325151085853577},{"id":"https://openalex.org/keywords/natural-language-generation","display_name":"Natural language generation","score":0.5041519403457642},{"id":"https://openalex.org/keywords/user-interface","display_name":"User interface","score":0.4944112300872803},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.49225321412086487},{"id":"https://openalex.org/keywords/software-portability","display_name":"Software portability","score":0.48587268590927124},{"id":"https://openalex.org/keywords/granularity","display_name":"Granularity","score":0.48397278785705566},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.45884501934051514},{"id":"https://openalex.org/keywords/natural-language-understanding","display_name":"Natural language understanding","score":0.4386489987373352},{"id":"https://openalex.org/keywords/natural","display_name":"Natural (archaeology)","score":0.4301775097846985},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.2778919041156769},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.11310437321662903}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.9202311038970947},{"id":"https://openalex.org/C174252522","wikidata":"https://www.wikidata.org/wiki/Q3816772","display_name":"Natural language user interface","level":3,"score":0.6417635679244995},{"id":"https://openalex.org/C2780719617","wikidata":"https://www.wikidata.org/wiki/Q1030752","display_name":"Salient","level":2,"score":0.6404463052749634},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.6383309960365295},{"id":"https://openalex.org/C113843644","wikidata":"https://www.wikidata.org/wiki/Q901882","display_name":"Interface (matter)","level":4,"score":0.5611523389816284},{"id":"https://openalex.org/C157657479","wikidata":"https://www.wikidata.org/wiki/Q2367247","display_name":"Closed captioning","level":3,"score":0.5407348871231079},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5325151085853577},{"id":"https://openalex.org/C2776187449","wikidata":"https://www.wikidata.org/wiki/Q1513879","display_name":"Natural language generation","level":3,"score":0.5041519403457642},{"id":"https://openalex.org/C89505385","wikidata":"https://www.wikidata.org/wiki/Q47146","display_name":"User interface","level":2,"score":0.4944112300872803},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.49225321412086487},{"id":"https://openalex.org/C63000827","wikidata":"https://www.wikidata.org/wiki/Q3080428","display_name":"Software portability","level":2,"score":0.48587268590927124},{"id":"https://openalex.org/C177774035","wikidata":"https://www.wikidata.org/wiki/Q1246948","display_name":"Granularity","level":2,"score":0.48397278785705566},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.45884501934051514},{"id":"https://openalex.org/C2779439875","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Natural language understanding","level":3,"score":0.4386489987373352},{"id":"https://openalex.org/C2776608160","wikidata":"https://www.wikidata.org/wiki/Q4785462","display_name":"Natural (archaeology)","level":2,"score":0.4301775097846985},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.2778919041156769},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.11310437321662903},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0},{"id":"https://openalex.org/C157915830","wikidata":"https://www.wikidata.org/wiki/Q2928001","display_name":"Bubble","level":2,"score":0.0},{"id":"https://openalex.org/C129307140","wikidata":"https://www.wikidata.org/wiki/Q6795880","display_name":"Maximum bubble pressure method","level":3,"score":0.0},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.0},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3564702","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3564702","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3564702","source":{"id":"https://openalex.org/S2492086750","display_name":"ACM Transactions on Intelligent Systems and Technology","issn_l":"2157-6904","issn":["2157-6904","2157-6912"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Intelligent Systems and Technology","raw_type":"journal-article"},{"id":"pmh:oai:aaltodoc.aalto.fi:123456789/120347","is_oa":true,"landing_page_url":"https://research.aalto.fi/en/publications/092efd3a-e7c7-4029-9aff-850168c9d67d","pdf_url":null,"source":{"id":"https://openalex.org/S4306401663","display_name":"Aaltodoc (Aalto University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I9927081","host_organization_name":"Aalto University","host_organization_lineage":["https://openalex.org/I9927081"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/article"},{"id":"pmh:oai:orbilu.uni.lu:10993/52281","is_oa":true,"landing_page_url":"http://orbilu.uni.lu/handle/10993/52281","pdf_url":null,"source":{"id":"https://openalex.org/S4306401815","display_name":"Open Repository and Bibliography (University of Luxembourg)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I186903577","host_organization_name":"University of Luxembourg","host_organization_lineage":["https://openalex.org/I186903577"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"ACM Transactions on Intelligent Systems and Technology (2022)","raw_type":"http://purl.org/coar/resource_type/c_6501"}],"best_oa_location":{"id":"doi:10.1145/3564702","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3564702","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3564702","source":{"id":"https://openalex.org/S2492086750","display_name":"ACM Transactions on Intelligent Systems and Technology","issn_l":"2157-6904","issn":["2157-6904","2157-6912"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Intelligent Systems and Technology","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.7799999713897705}],"awards":[{"id":"https://openalex.org/G3544863752","display_name":null,"funder_award_id":"318559","funder_id":"https://openalex.org/F4320321108","funder_display_name":"Academy of Finland"},{"id":"https://openalex.org/G5078460471","display_name":null,"funder_award_id":"310947","funder_id":"https://openalex.org/F4320321108","funder_display_name":"Academy of Finland"},{"id":"https://openalex.org/G7244745199","display_name":null,"funder_award_id":"CHIST-ERA-20-BCI-001","funder_id":"https://openalex.org/F4320321108","funder_display_name":"Academy of Finland"},{"id":"https://openalex.org/G7561857167","display_name":null,"funder_award_id":"CHIST-ERA-20-BCI-001","funder_id":"https://openalex.org/F4320338463","funder_display_name":"CHIST-ERA"},{"id":"https://openalex.org/G827859981","display_name":null,"funder_award_id":"291556, 318559, 310947","funder_id":"https://openalex.org/F4320321108","funder_display_name":"Academy of Finland"},{"id":"https://openalex.org/G8488328798","display_name":null,"funder_award_id":"291556","funder_id":"https://openalex.org/F4320321108","funder_display_name":"Academy of Finland"}],"funders":[{"id":"https://openalex.org/F4320320300","display_name":"European Commission","ror":"https://ror.org/00k4n6c32"},{"id":"https://openalex.org/F4320321108","display_name":"Academy of Finland","ror":"https://ror.org/05k73zm37"},{"id":"https://openalex.org/F4320338448","display_name":"HORIZON EUROPE European Innovation Council","ror":null},{"id":"https://openalex.org/F4320338463","display_name":"CHIST-ERA","ror":"https://ror.org/00rbzpz17"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4297220110.pdf","grobid_xml":"https://content.openalex.org/works/W4297220110.grobid-xml"},"referenced_works_count":67,"referenced_works":["https://openalex.org/W1861492603","https://openalex.org/W1897761818","https://openalex.org/W1931639407","https://openalex.org/W1964157254","https://openalex.org/W1971443254","https://openalex.org/W1992365067","https://openalex.org/W2066134726","https://openalex.org/W2066380143","https://openalex.org/W2070016268","https://openalex.org/W2090048052","https://openalex.org/W2092939357","https://openalex.org/W2095759914","https://openalex.org/W2106338250","https://openalex.org/W2106652774","https://openalex.org/W2108598243","https://openalex.org/W2115063264","https://openalex.org/W2116716943","https://openalex.org/W2120792663","https://openalex.org/W2126014052","https://openalex.org/W2129410131","https://openalex.org/W2135957164","https://openalex.org/W2141125339","https://openalex.org/W2158447593","https://openalex.org/W2162939096","https://openalex.org/W2171581899","https://openalex.org/W2194775991","https://openalex.org/W2277195237","https://openalex.org/W2295423240","https://openalex.org/W2302086703","https://openalex.org/W2325920097","https://openalex.org/W2406390611","https://openalex.org/W2568262903","https://openalex.org/W2575842049","https://openalex.org/W2594440402","https://openalex.org/W2604799547","https://openalex.org/W2616247523","https://openalex.org/W2739046565","https://openalex.org/W2745461083","https://openalex.org/W2754927243","https://openalex.org/W2765874585","https://openalex.org/W2808599441","https://openalex.org/W2889009749","https://openalex.org/W2891503716","https://openalex.org/W2896348597","https://openalex.org/W2897966957","https://openalex.org/W2963091658","https://openalex.org/W2963095307","https://openalex.org/W2963206148","https://openalex.org/W2963912046","https://openalex.org/W2967223102","https://openalex.org/W2970383515","https://openalex.org/W2981526789","https://openalex.org/W2998746484","https://openalex.org/W3029105705","https://openalex.org/W3035663944","https://openalex.org/W3091272181","https://openalex.org/W3091588028","https://openalex.org/W3101041928","https://openalex.org/W3102564565","https://openalex.org/W3124386704","https://openalex.org/W3134648203","https://openalex.org/W3163374010","https://openalex.org/W3174476431","https://openalex.org/W4211222216","https://openalex.org/W4234617775","https://openalex.org/W4248191435","https://openalex.org/W4301846271"],"related_works":["https://openalex.org/W3034800807","https://openalex.org/W3035130722","https://openalex.org/W4287758468","https://openalex.org/W4310731959","https://openalex.org/W3102948683","https://openalex.org/W3200659790","https://openalex.org/W2386843486","https://openalex.org/W3092480528","https://openalex.org/W4285240985","https://openalex.org/W1711541733"],"abstract_inverted_index":{"Being":[0],"able":[1],"to":[2,44,85,116,124],"describe":[3,92,118],"any":[4],"user":[5,101],"interface":[6,97],"(UI)":[7],"screenshot":[8],"in":[9,95],"natural":[10,76],"language":[11,77],"can":[12,102],"promote":[13],"understanding":[14],"of":[15,19,48],"the":[16,20,40,60,96,100,119],"main":[17],"purpose":[18],"UI,":[21,120],"yet":[22],"currently":[23],"it":[24],"cannot":[25],"be":[26],"accomplished":[27],"with":[28,51,83,104],"state-of-the-art":[29],"captioning":[30],"systems.":[31],"We":[32,106],"introduce":[33],"XUI,":[34],"a":[35],"novel":[36],"method":[37],"inspired":[38],"by":[39],"global":[41],"precedence":[42],"effect":[43],"create":[45],"informative":[46],"descriptions":[47,58,82,110],"UIs,":[49],"starting":[50],"an":[52],"overview":[53],"and":[54,75,98,121],"then":[55],"providing":[56],"fine-grained":[57],"about":[59],"most":[61],"salient":[62],"elements.":[63],"XUI":[64,80,109,128],"builds":[65],"upon":[66],"computational":[67],"models":[68],"for":[69],"topic":[70],"classification,":[71],"visual":[72],"saliency":[73],"prediction,":[74],"generation":[78],"(NLG).":[79],"provides":[81],"up":[84],"three":[86],"different":[87],"granularity":[88],"levels":[89],"that,":[90],"together,":[91],"what":[93,99],"is":[94,129],"do":[103],"it.":[105],"found":[107],"that":[108],"are":[111,114],"highly":[112],"readable,":[113],"perceived":[115],"accurately":[117],"score":[122],"similarly":[123],"human-generated":[125],"UI":[126],"descriptions.":[127],"available":[130],"as":[131],"open-source":[132],"software.":[133]},"counts_by_year":[{"year":2025,"cited_by_count":8},{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":3}],"updated_date":"2026-07-20T07:56:41.581041","created_date":"2025-10-10T00:00:00"}