{"id":"https://openalex.org/W4220747226","doi":"https://doi.org/10.1145/3490099.3511109","title":"Understanding Screen Relationships from Screenshots of Smartphone Applications","display_name":"Understanding Screen Relationships from Screenshots of Smartphone Applications","publication_year":2022,"publication_date":"2022-03-21","ids":{"openalex":"https://openalex.org/W4220747226","doi":"https://doi.org/10.1145/3490099.3511109"},"language":"en","primary_location":{"id":"doi:10.1145/3490099.3511109","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3490099.3511109","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3490099.3511109","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"27th International Conference on Intelligent User Interfaces","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3490099.3511109","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5021348625","display_name":"Shirin Feiz","orcid":"https://orcid.org/0000-0001-6054-7305"},"institutions":[{"id":"https://openalex.org/I59553526","display_name":"Stony Brook University","ror":"https://ror.org/05qghxh33","country_code":"US","type":"education","lineage":["https://openalex.org/I59553526"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Shirin Feiz","raw_affiliation_strings":["Computer Science department, Stony Brook University, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Computer Science department, Stony Brook University, United States","institution_ids":["https://openalex.org/I59553526"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076580255","display_name":"Jason Wu","orcid":"https://orcid.org/0000-0001-5101-0557"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jason Wu","raw_affiliation_strings":["Human-Computer Interaction Institute, Carnegie Mellon University, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Human-Computer Interaction Institute, Carnegie Mellon University, United States","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100357469","display_name":"Xiaoyi Zhang","orcid":"https://orcid.org/0000-0002-7185-0470"},"institutions":[{"id":"https://openalex.org/I4210153776","display_name":"Apple (United States)","ror":"https://ror.org/059hsda18","country_code":"US","type":"company","lineage":["https://openalex.org/I4210153776"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xiaoyi Zhang","raw_affiliation_strings":["Apple Inc, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Apple Inc, United States","institution_ids":["https://openalex.org/I4210153776"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085693746","display_name":"Amanda Swearngin","orcid":"https://orcid.org/0000-0003-2468-6337"},"institutions":[{"id":"https://openalex.org/I4210153776","display_name":"Apple (United States)","ror":"https://ror.org/059hsda18","country_code":"US","type":"company","lineage":["https://openalex.org/I4210153776"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Amanda Swearngin","raw_affiliation_strings":["Apple, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Apple, United States","institution_ids":["https://openalex.org/I4210153776"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060033622","display_name":"Titus Barik","orcid":"https://orcid.org/0000-0002-4877-0739"},"institutions":[{"id":"https://openalex.org/I4210153776","display_name":"Apple (United States)","ror":"https://ror.org/059hsda18","country_code":"US","type":"company","lineage":["https://openalex.org/I4210153776"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Titus Barik","raw_affiliation_strings":["Apple, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Apple, United States","institution_ids":["https://openalex.org/I4210153776"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5081566660","display_name":"Jeffrey Nichols","orcid":"https://orcid.org/0000-0002-6880-8546"},"institutions":[{"id":"https://openalex.org/I4210153776","display_name":"Apple (United States)","ror":"https://ror.org/059hsda18","country_code":"US","type":"company","lineage":["https://openalex.org/I4210153776"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jeffrey Nichols","raw_affiliation_strings":["Apple, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Apple, United States","institution_ids":["https://openalex.org/I4210153776"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5021348625"],"corresponding_institution_ids":["https://openalex.org/I59553526"],"apc_list":null,"apc_paid":null,"fwci":8.2948,"has_fulltext":true,"cited_by_count":27,"citation_normalized_percentile":{"value":0.97570361,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"447","last_page":"458"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.9968000054359436,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.9968000054359436,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11241","display_name":"Advanced Malware Detection Techniques","score":0.9952999949455261,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12238","display_name":"Green IT and Sustainability","score":0.9714000225067139,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/scrolling","display_name":"Scrolling","score":0.8612440228462219},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7791900634765625},{"id":"https://openalex.org/keywords/crawling","display_name":"Crawling","score":0.6956807971000671},{"id":"https://openalex.org/keywords/web-crawler","display_name":"Web crawler","score":0.544338047504425},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.5135126709938049},{"id":"https://openalex.org/keywords/macro","display_name":"Macro","score":0.4548611342906952},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.4523051083087921},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.4199870824813843},{"id":"https://openalex.org/keywords/display-size","display_name":"Display size","score":0.41510677337646484},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.38725394010543823},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.2633748948574066},{"id":"https://openalex.org/keywords/display-device","display_name":"Display device","score":0.13123410940170288},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.11263641715049744}],"concepts":[{"id":"https://openalex.org/C59046462","wikidata":"https://www.wikidata.org/wiki/Q181319","display_name":"Scrolling","level":2,"score":0.8612440228462219},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7791900634765625},{"id":"https://openalex.org/C100368936","wikidata":"https://www.wikidata.org/wiki/Q1411725","display_name":"Crawling","level":2,"score":0.6956807971000671},{"id":"https://openalex.org/C13743948","wikidata":"https://www.wikidata.org/wiki/Q45842","display_name":"Web crawler","level":2,"score":0.544338047504425},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.5135126709938049},{"id":"https://openalex.org/C166955791","wikidata":"https://www.wikidata.org/wiki/Q629579","display_name":"Macro","level":2,"score":0.4548611342906952},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.4523051083087921},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.4199870824813843},{"id":"https://openalex.org/C170003942","wikidata":"https://www.wikidata.org/wiki/Q861066","display_name":"Display size","level":3,"score":0.41510677337646484},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.38725394010543823},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2633748948574066},{"id":"https://openalex.org/C164597639","wikidata":"https://www.wikidata.org/wiki/Q327065","display_name":"Display device","level":2,"score":0.13123410940170288},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.11263641715049744},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.0},{"id":"https://openalex.org/C105702510","wikidata":"https://www.wikidata.org/wiki/Q514","display_name":"Anatomy","level":1,"score":0.0},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3490099.3511109","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3490099.3511109","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3490099.3511109","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"27th International Conference on Intelligent User Interfaces","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3490099.3511109","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3490099.3511109","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3490099.3511109","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"27th International Conference on Intelligent User Interfaces","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4220747226.pdf","grobid_xml":"https://content.openalex.org/works/W4220747226.grobid-xml"},"referenced_works_count":32,"referenced_works":["https://openalex.org/W95608104","https://openalex.org/W1812630525","https://openalex.org/W2101800210","https://openalex.org/W2194775991","https://openalex.org/W2533771361","https://openalex.org/W2597507805","https://openalex.org/W2599566598","https://openalex.org/W2765874585","https://openalex.org/W2802838214","https://openalex.org/W2804434745","https://openalex.org/W2808991421","https://openalex.org/W2883757633","https://openalex.org/W2897267527","https://openalex.org/W2897966957","https://openalex.org/W2899070402","https://openalex.org/W2909014315","https://openalex.org/W2942422815","https://openalex.org/W2962992787","https://openalex.org/W2979382951","https://openalex.org/W2985406956","https://openalex.org/W2999907851","https://openalex.org/W3034336960","https://openalex.org/W3090530797","https://openalex.org/W3124240085","https://openalex.org/W3134648203","https://openalex.org/W3160022552","https://openalex.org/W3160987621","https://openalex.org/W3163374010","https://openalex.org/W3177754813","https://openalex.org/W3194555047","https://openalex.org/W3199730823","https://openalex.org/W3205557064"],"related_works":["https://openalex.org/W2566658409","https://openalex.org/W3119324922","https://openalex.org/W2352686120","https://openalex.org/W2372594123","https://openalex.org/W2358310581","https://openalex.org/W2964752624","https://openalex.org/W2026132847","https://openalex.org/W4385695127","https://openalex.org/W2137810919","https://openalex.org/W4255854114"],"abstract_inverted_index":{"All":[0],"graphical":[1],"user":[2,17],"interfaces":[3],"are":[4],"comprised":[5],"of":[6,25,32,55,100,113,150,185,198,208,246,253,278,303,319],"one":[7],"or":[8,123,127,174,176,288,293],"more":[9,211],"screens":[10,24,38,187],"that":[11,34,43,88,103,221,232,265,280],"may":[12,154,192],"be":[13,45],"shown":[14],"to":[15,72,135,163,216,243,271],"the":[16,30,37,75,82,101,111,116,151,181,189,247,286,306,325],"depending":[18],"on":[19,36,98,115,305,311],"their":[20],"interactions.":[21],"Identifying":[22],"different":[23,164,186,225],"an":[26,67,132,283,300,315],"app":[27,52,56,62,69,102,191],"and":[28,59,157,201,259,275,295,314],"understanding":[29],"type":[31,112],"changes":[33],"happen":[35],"is":[39,78,81,129,144],"a":[40,145,167,206,228,234,239,251,256,261,267,289],"challenging":[41],"task":[42,147],"can":[44,94],"applied":[46],"in":[47,166,196,224,282,324],"many":[48],"areas":[49],"including":[50],"automatic":[51,133],"crawling,":[53],"playback":[54],"automation":[57],"macros":[58],"large":[60],"scale":[61],"dataset":[63,207],"analysis.":[64],"For":[65],"example,":[66],"automated":[68],"crawler":[70,134],"needs":[71],"understand":[73,222],"if":[74],"screen":[76,87,142,153,229,249,262,307],"it":[77,89,93,104],"currently":[79],"viewing":[80],"same":[83,152,182,190,248],"as":[84,119,148],"any":[85,121],"previous":[86],"has":[90,105],"encountered,":[91],"so":[92],"focus":[95],"its":[96],"efforts":[97],"portions":[99],"not":[106],"yet":[107],"explored.":[108],"Moreover,":[109],"identifying":[110],"change":[114],"screen,":[117],"such":[118,137],"whether":[120],"dialogues":[122],"keyboards":[124],"have":[125,155],"opened":[126],"closed,":[128],"useful":[130],"for":[131,160],"handle":[136],"events":[138,279,323],"while":[139],"crawling.":[140],"Understanding":[141],"relationships":[143],"difficult":[146],"instances":[149,184,245],"visual":[156],"structural":[158],"variation,":[159],"example":[161],"due":[162],"content":[165,177],"database-backed":[168],"application,":[169],"scrolling,":[170],"dialog":[171,290],"boxes":[172],"opening":[173],"closing,":[175],"loading":[178],"delays.":[179],"At":[180],"time,":[183],"from":[188,210,250,255],"share":[193],"some":[194],"similarities":[195],"terms":[197],"design,":[199],"structure,":[200],"content.":[202],"This":[203],"paper":[204],"uses":[205,266],"screenshots":[209,254],"than":[212],"1K":[213],"iPhone":[214],"applications":[215],"train":[217],"two":[218],"ML":[219],"models":[220,298],"similarity":[223,230,274,308],"ways:":[226],"(1)":[227],"model":[231,241,264],"combines":[233],"UI":[235],"object":[236],"detector":[237],"with":[238],"transformer":[240],"architecture":[242,270],"recognize":[244],"collection":[252],"single":[257],"app,":[258],"(2)":[260],"transition":[263,326],"siamese":[268],"network":[269],"identify":[272],"both":[273],"three":[276],"types":[277],"appear":[281],"interaction":[284],"trace:":[285],"keyboard":[287],"box":[291],"appearing":[292],"disappearing,":[294],"scrolling.":[296],"Our":[297],"achieve":[299],"F1":[301,317],"score":[302,318],"0.83":[304],"task,":[309],"improving":[310],"comparable":[312],"baselines,":[313],"average":[316],"0.71":[320],"across":[321],"all":[322],"task.":[327]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":9},{"year":2024,"cited_by_count":8},{"year":2023,"cited_by_count":8},{"year":2022,"cited_by_count":1}],"updated_date":"2026-05-05T08:41:31.759640","created_date":"2025-10-10T00:00:00"}