{"id":"https://openalex.org/W2138917250","doi":"https://doi.org/10.1145/2047196.2047228","title":"Associating the visual representation of user interfaces with their internal structures and metadata","display_name":"Associating the visual representation of user interfaces with their internal structures and metadata","publication_year":2011,"publication_date":"2011-10-16","ids":{"openalex":"https://openalex.org/W2138917250","doi":"https://doi.org/10.1145/2047196.2047228","mag":"2138917250"},"language":"en","primary_location":{"id":"doi:10.1145/2047196.2047228","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2047196.2047228","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 24th annual ACM symposium on User interface software and technology","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://hdl.handle.net/1721.1/121428","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5002765714","display_name":"Tsung-Hsiang Chang","orcid":null},"institutions":[{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tsung-Hsiang Chang","raw_affiliation_strings":["Massachusetts Institute of Technology, Cambridge, MA, USA","Massachusetts Institute of Technology; ,; Cambridge MA USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Massachusetts Institute of Technology, Cambridge, MA, USA","institution_ids":["https://openalex.org/I63966007"]},{"raw_affiliation_string":"Massachusetts Institute of Technology; ,; Cambridge MA USA","institution_ids":["https://openalex.org/I63966007"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070687718","display_name":"Tom Yeh","orcid":"https://orcid.org/0000-0003-0227-9781"},"institutions":[{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tom Yeh","raw_affiliation_strings":["University of Maryland, College Park, MD, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Maryland, College Park, MD, USA","institution_ids":["https://openalex.org/I66946132"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5000689705","display_name":"Rob Miller","orcid":null},"institutions":[{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Rob Miller","raw_affiliation_strings":["Massachusetts Institute of Technology, Cambridge, MA, USA","Massachusetts Institute of Technology; ,; Cambridge MA USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Massachusetts Institute of Technology, Cambridge, MA, USA","institution_ids":["https://openalex.org/I63966007"]},{"raw_affiliation_string":"Massachusetts Institute of Technology; ,; Cambridge MA USA","institution_ids":["https://openalex.org/I63966007"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":48,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"245","last_page":"256"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10789","display_name":"Interactive and Immersive Displays","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10789","display_name":"Interactive and Immersive Displays","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12481","display_name":"Digital Accessibility for Disabilities","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/3307","display_name":"Human Factors and Ergonomics"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10914","display_name":"Tactile and Sensory Interactions","score":0.996399998664856,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8470990657806396},{"id":"https://openalex.org/keywords/metadata","display_name":"Metadata","score":0.7792937755584717},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.5549915432929993},{"id":"https://openalex.org/keywords/implementation","display_name":"Implementation","score":0.5488041043281555},{"id":"https://openalex.org/keywords/pixel","display_name":"Pixel","score":0.481810986995697},{"id":"https://openalex.org/keywords/readability","display_name":"Readability","score":0.4749981164932251},{"id":"https://openalex.org/keywords/user-interface","display_name":"User interface","score":0.44527536630630493},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.42329341173171997},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.37103646993637085},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.28907862305641174},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.204039067029953},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.16710779070854187}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8470990657806396},{"id":"https://openalex.org/C93518851","wikidata":"https://www.wikidata.org/wiki/Q180160","display_name":"Metadata","level":2,"score":0.7792937755584717},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.5549915432929993},{"id":"https://openalex.org/C26713055","wikidata":"https://www.wikidata.org/wiki/Q245962","display_name":"Implementation","level":2,"score":0.5488041043281555},{"id":"https://openalex.org/C160633673","wikidata":"https://www.wikidata.org/wiki/Q355198","display_name":"Pixel","level":2,"score":0.481810986995697},{"id":"https://openalex.org/C2778143727","wikidata":"https://www.wikidata.org/wiki/Q1820650","display_name":"Readability","level":2,"score":0.4749981164932251},{"id":"https://openalex.org/C89505385","wikidata":"https://www.wikidata.org/wiki/Q47146","display_name":"User interface","level":2,"score":0.44527536630630493},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.42329341173171997},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.37103646993637085},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.28907862305641174},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.204039067029953},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.16710779070854187},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/2047196.2047228","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2047196.2047228","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 24th annual ACM symposium on User interface software and technology","raw_type":"proceedings-article"},{"id":"pmh:oai:dspace.mit.edu:1721.1/121428","is_oa":true,"landing_page_url":"https://hdl.handle.net/1721.1/121428","pdf_url":null,"source":{"id":"https://openalex.org/S4306400425","display_name":"DSpace@MIT (Massachusetts Institute of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I63966007","host_organization_name":"Massachusetts Institute of Technology","host_organization_lineage":["https://openalex.org/I63966007"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"MIT web domain","raw_type":"http://purl.org/eprint/type/ConferencePaper"}],"best_oa_location":{"id":"pmh:oai:dspace.mit.edu:1721.1/121428","is_oa":true,"landing_page_url":"https://hdl.handle.net/1721.1/121428","pdf_url":null,"source":{"id":"https://openalex.org/S4306400425","display_name":"DSpace@MIT (Massachusetts Institute of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I63966007","host_organization_name":"Massachusetts Institute of Technology","host_organization_lineage":["https://openalex.org/I63966007"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"MIT web domain","raw_type":"http://purl.org/eprint/type/ConferencePaper"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G3869116009","display_name":null,"funder_award_id":"IIS-0447800","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G6999320186","display_name":"CAREER:    End-user Programming for Web Automation and Visualization","funder_award_id":"0447800","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":19,"referenced_works":["https://openalex.org/W105942870","https://openalex.org/W1523479945","https://openalex.org/W2040244898","https://openalex.org/W2044054519","https://openalex.org/W2051593684","https://openalex.org/W2071024706","https://openalex.org/W2071965495","https://openalex.org/W2078091018","https://openalex.org/W2115063264","https://openalex.org/W2135345119","https://openalex.org/W2141125339","https://openalex.org/W2143058502","https://openalex.org/W2145376354","https://openalex.org/W2148384385","https://openalex.org/W2158447593","https://openalex.org/W2163534337","https://openalex.org/W2341994895","https://openalex.org/W2399954834","https://openalex.org/W6681242512"],"related_works":["https://openalex.org/W1964661231","https://openalex.org/W4254960163","https://openalex.org/W3110264473","https://openalex.org/W2032810564","https://openalex.org/W2370831213","https://openalex.org/W2972807713","https://openalex.org/W4315704968","https://openalex.org/W2100255360","https://openalex.org/W2107506952","https://openalex.org/W3135287016"],"abstract_inverted_index":{"Pixel-based":[0],"methods":[1],"are":[2],"emerging":[3],"as":[4,36],"a":[5,47,79,101],"new":[6,12,87],"and":[7,62,71,84,98,126,146],"promising":[8],"way":[9],"to":[10,24,76,149,156],"develop":[11,83],"interaction":[13],"techniques":[14],"on":[15,95],"top":[16],"of":[17,56,115,130,158],"existing":[18,121],"user":[19,57],"interfaces.":[20],"However,":[21],"in":[22,113],"order":[23],"maintain":[25],"platform":[26],"independence,":[27],"other":[28],"available":[29],"low-level":[30],"information":[31],"about":[32],"GUI":[33],"widgets,":[34],"such":[35,78],"accessibility":[37],"metadata,":[38],"was":[39],"neglected":[40],"intentionally.":[41],"In":[42],"this":[43],"paper,":[44],"we":[45,109,139],"present":[46],"hybrid":[48],"framework,":[49],"PAX,":[50],"which":[51],"associates":[52],"the":[53,60,68,96,128,135],"visual":[54],"representation":[55],"interfaces":[58],"(i.e.":[59,67],"pixels)":[61],"their":[63],"internal":[64],"hierarchical":[65],"metadata":[66],"content,":[69],"role,":[70],"value).":[72],"We":[73,81,118],"identify":[74],"challenges":[75],"building":[77],"framework.":[80],"also":[82],"evaluate":[85],"two":[86,141],"algorithms":[88],"for":[89,99],"detecting":[90],"text":[91,102],"at":[92,134],"arbitrary":[93],"places":[94],"screen,":[97],"segmenting":[100],"image":[103],"into":[104],"individual":[105],"word":[106],"blobs.":[107],"Finally,":[108],"validate":[110],"our":[111],"framework":[112],"implementations":[114],"three":[116],"applications.":[117],"enhance":[119],"an":[120],"pixel-based":[122],"system,":[123],"Sikuli":[124],"Script,":[125],"preserve":[127],"readability":[129],"its":[131],"script":[132],"code":[133],"same":[136],"time.":[137],"Further,":[138],"create":[140],"novel":[142],"applications,":[143],"Screen":[144,147],"Search":[145],"Copy,":[148],"demonstrate":[150],"how":[151],"PAX":[152],"can":[153],"be":[154],"applied":[155],"development":[157],"desktop-level":[159],"interactive":[160],"systems.":[161]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":2},{"year":2021,"cited_by_count":4},{"year":2020,"cited_by_count":2},{"year":2019,"cited_by_count":6},{"year":2018,"cited_by_count":7},{"year":2017,"cited_by_count":7},{"year":2016,"cited_by_count":2},{"year":2015,"cited_by_count":4},{"year":2014,"cited_by_count":3},{"year":2013,"cited_by_count":3},{"year":2012,"cited_by_count":4}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}