{"id":"https://openalex.org/W4239883684","doi":"https://doi.org/10.1109/icse.2013.6606669","title":"Exploring the internal state of user interfaces by combining computer vision techniques with grammatical inference","display_name":"Exploring the internal state of user interfaces by combining computer vision techniques with grammatical inference","publication_year":2013,"publication_date":"2013-05-01","ids":{"openalex":"https://openalex.org/W4239883684","doi":"https://doi.org/10.1109/icse.2013.6606669"},"language":"en","primary_location":{"id":"doi:10.1109/icse.2013.6606669","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icse.2013.6606669","pdf_url":null,"source":{"id":"https://openalex.org/S4363607951","display_name":"2013 35th International Conference on Software Engineering (ICSE)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2013 35th International Conference on Software Engineering (ICSE)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5086779019","display_name":"Paul Givens","orcid":null},"institutions":[{"id":"https://openalex.org/I188538660","display_name":"University of Colorado Boulder","ror":"https://ror.org/02ttsq026","country_code":"US","type":"education","lineage":["https://openalex.org/I188538660"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Paul Givens","raw_affiliation_strings":["University of Colorado Boulder, Boulder, CO, US"],"affiliations":[{"raw_affiliation_string":"University of Colorado Boulder, Boulder, CO, US","institution_ids":["https://openalex.org/I188538660"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5044253018","display_name":"Aleksandar Chakarov","orcid":null},"institutions":[{"id":"https://openalex.org/I188538660","display_name":"University of Colorado Boulder","ror":"https://ror.org/02ttsq026","country_code":"US","type":"education","lineage":["https://openalex.org/I188538660"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Aleksandar Chakarov","raw_affiliation_strings":["University of Colorado Boulder, Boulder, CO, US"],"affiliations":[{"raw_affiliation_string":"University of Colorado Boulder, Boulder, CO, US","institution_ids":["https://openalex.org/I188538660"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103106502","display_name":"Sriram Sankaranarayanan","orcid":"https://orcid.org/0000-0001-7315-4340"},"institutions":[{"id":"https://openalex.org/I188538660","display_name":"University of Colorado Boulder","ror":"https://ror.org/02ttsq026","country_code":"US","type":"education","lineage":["https://openalex.org/I188538660"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sriram Sankaranarayanan","raw_affiliation_strings":["University of Colorado Boulder, Boulder, CO, US"],"affiliations":[{"raw_affiliation_string":"University of Colorado Boulder, Boulder, CO, US","institution_ids":["https://openalex.org/I188538660"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5070687718","display_name":"Tom Yeh","orcid":"https://orcid.org/0000-0003-0227-9781"},"institutions":[{"id":"https://openalex.org/I188538660","display_name":"University of Colorado Boulder","ror":"https://ror.org/02ttsq026","country_code":"US","type":"education","lineage":["https://openalex.org/I188538660"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tom Yeh","raw_affiliation_strings":["University of Colorado Boulder, Boulder, CO, US"],"affiliations":[{"raw_affiliation_string":"University of Colorado Boulder, Boulder, CO, US","institution_ids":["https://openalex.org/I188538660"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5086779019"],"corresponding_institution_ids":["https://openalex.org/I188538660"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":4,"citation_normalized_percentile":{"value":0.4746349,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":94},"biblio":{"volume":"6339","issue":null,"first_page":"1165","last_page":"1168"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.9957000017166138,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.9929999709129333,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7891725897789001},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.6885549426078796},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5298343300819397},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.4607122838497162},{"id":"https://openalex.org/keywords/state","display_name":"State (computer science)","score":0.458365261554718},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.43936020135879517},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.1611270308494568}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7891725897789001},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.6885549426078796},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5298343300819397},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4607122838497162},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.458365261554718},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.43936020135879517},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.1611270308494568}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icse.2013.6606669","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icse.2013.6606669","pdf_url":null,"source":{"id":"https://openalex.org/S4363607951","display_name":"2013 35th International Conference on Software Engineering (ICSE)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2013 35th International Conference on Software Engineering (ICSE)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.699999988079071,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[{"id":"https://openalex.org/G4472538925","display_name":"CPS: Small: Formal Analysis of Man-Machine Interfaces to Cyber-Physical Systems","funder_award_id":"1035845","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G4713059963","display_name":null,"funder_award_id":"FA8750","funder_id":"https://openalex.org/F4320332180","funder_display_name":"Defense Advanced Research Projects Agency"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320332180","display_name":"Defense Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":24,"referenced_works":["https://openalex.org/W1594272399","https://openalex.org/W1598996260","https://openalex.org/W2013887890","https://openalex.org/W2017603160","https://openalex.org/W2061079066","https://openalex.org/W2108825163","https://openalex.org/W2110107911","https://openalex.org/W2118518941","https://openalex.org/W2123591057","https://openalex.org/W2132177520","https://openalex.org/W2136518234","https://openalex.org/W2141125339","https://openalex.org/W2141811956","https://openalex.org/W2143058502","https://openalex.org/W2146105230","https://openalex.org/W2148334620","https://openalex.org/W2295399529","https://openalex.org/W4241372395","https://openalex.org/W4250359162","https://openalex.org/W6635673997","https://openalex.org/W6635840230","https://openalex.org/W6677359323","https://openalex.org/W6678243694","https://openalex.org/W6680904998"],"related_works":["https://openalex.org/W2055243143","https://openalex.org/W4321636575","https://openalex.org/W1986418932","https://openalex.org/W2357796999","https://openalex.org/W2045526782","https://openalex.org/W2741131631","https://openalex.org/W2156919374","https://openalex.org/W1984019423","https://openalex.org/W4280588203","https://openalex.org/W3204019825"],"abstract_inverted_index":{"In":[0],"this":[1],"paper,":[2],"we":[3],"present":[4],"a":[5,16,28,89,139,170,178],"promising":[6],"approach":[7,118],"to":[8,34,101,122,156],"systematically":[9],"testing":[10],"graphical":[11,37],"user":[12,93],"interfaces":[13,94,102,173],"(GUI)":[14],"in":[15,39,128],"platform":[17],"independent":[18],"manner.":[19],"Our":[20,86],"framework":[21,87,164],"uses":[22],"standard":[23],"computer":[24],"vision":[25],"techniques":[26,69],"through":[27],"python-based":[29],"scripting":[30],"language":[31],"(Sikuli":[32],"script)":[33],"identify":[35],"key":[36],"elements":[38,47],"the":[40,62,73,80,84,113,129,144,153,163,175],"screen":[41],"and":[42,51,58,77,110,147,165],"automatically":[43],"interact":[44],"with":[45],"these":[46],"by":[48,116,188],"simulating":[49],"keypresses":[50],"pointer":[52],"clicks.":[53],"The":[54],"sequence":[55],"of":[56,79,92,162,172,177],"inputs":[57],"outputs":[59],"resulting":[60],"from":[61,96],"interaction":[63],"is":[64,185],"analyzed":[65],"using":[66],"grammatical":[67],"inference":[68],"that":[70,150,184],"can":[71,119,151],"infer":[72],"likely":[74],"internal":[75,131],"states":[76],"transitions":[78],"GUI":[81],"based":[82],"on":[83,169],"observations.":[85],"handles":[88],"wide":[90],"variety":[91,171],"ranging":[95],"traditional":[97],"pull":[98],"down":[99],"menus":[100],"built":[103],"for":[104,124],"mobile":[105],"platforms":[106],"such":[107,134],"as":[108,135],"Android":[109],"iOS.":[111],"Furthermore,":[112],"automaton":[114],"inferred":[115],"our":[117],"be":[120],"used":[121,187],"check":[123],"potentially":[125],"harmful":[126],"patterns":[127],"interface's":[130],"state":[132],"machine":[133],"design":[136],"inconsistencies":[137],"(eg,.":[138],"keypress":[140],"does":[141],"not":[142],"have":[143],"intended":[145],"effect)":[146],"mode":[148],"confusion":[149],"make":[152],"interface":[154],"hard":[155],"use.":[157],"We":[158],"describe":[159],"an":[160],"implementation":[161],"demonstrate":[166],"its":[167],"working":[168],"including":[174],"user-interface":[176],"safety":[179],"critical":[180],"insulin":[181],"infusion":[182],"pump":[183],"commonly":[186],"type-1":[189],"diabetic":[190],"patients.":[191]},"counts_by_year":[{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":1},{"year":2018,"cited_by_count":1},{"year":2017,"cited_by_count":1}],"updated_date":"2026-04-21T08:09:41.155169","created_date":"2025-10-10T00:00:00"}