{"id":"https://openalex.org/W2951165326","doi":"https://doi.org/10.1109/tr.2020.2972266","title":"METTLE: A METamorphic Testing Approach to Assessing and Validating Unsupervised Machine Learning Systems","display_name":"METTLE: A METamorphic Testing Approach to Assessing and Validating Unsupervised Machine Learning Systems","publication_year":2020,"publication_date":"2020-03-13","ids":{"openalex":"https://openalex.org/W2951165326","doi":"https://doi.org/10.1109/tr.2020.2972266","mag":"2951165326"},"language":"en","primary_location":{"id":"doi:10.1109/tr.2020.2972266","is_oa":true,"landing_page_url":"https://doi.org/10.1109/tr.2020.2972266","pdf_url":"https://ieeexplore.ieee.org/ielx7/24/9274445/09036058.pdf","source":{"id":"https://openalex.org/S87725633","display_name":"IEEE Transactions on Reliability","issn_l":"0018-9529","issn":["0018-9529","1558-1721"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Reliability","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://ieeexplore.ieee.org/ielx7/24/9274445/09036058.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Xiaoyuan Xie","orcid":"https://orcid.org/0000-0003-2828-7165"},"institutions":[{"id":"https://openalex.org/I37461747","display_name":"Wuhan University","ror":"https://ror.org/033vjfk17","country_code":"CN","type":"education","lineage":["https://openalex.org/I37461747"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Xiaoyuan Xie","raw_affiliation_strings":["School of Computer Science, Wuhan University, Wuhan, China"],"affiliations":[{"raw_affiliation_string":"School of Computer Science, Wuhan University, Wuhan, China","institution_ids":["https://openalex.org/I37461747"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Zhiyi Zhang","orcid":"https://orcid.org/0000-0002-0029-0912"},"institutions":[{"id":"https://openalex.org/I37461747","display_name":"Wuhan University","ror":"https://ror.org/033vjfk17","country_code":"CN","type":"education","lineage":["https://openalex.org/I37461747"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhiyi Zhang","raw_affiliation_strings":["School of Computer Science, Wuhan University, Wuhan, China"],"affiliations":[{"raw_affiliation_string":"School of Computer Science, Wuhan University, Wuhan, China","institution_ids":["https://openalex.org/I37461747"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Tsong Yueh Chen","orcid":"https://orcid.org/0000-0003-3578-0994"},"institutions":[{"id":"https://openalex.org/I57093077","display_name":"Swinburne University of Technology","ror":"https://ror.org/031rekg67","country_code":"AU","type":"education","lineage":["https://openalex.org/I57093077"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Tsong Yueh Chen","raw_affiliation_strings":["Department of Computer Science and Software Engineering, Swinburne University of Technology, Hawthorn, VIC, Australia"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Software Engineering, Swinburne University of Technology, Hawthorn, VIC, Australia","institution_ids":["https://openalex.org/I57093077"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Yang Liu","orcid":"https://orcid.org/0000-0001-7300-9215"},"institutions":[{"id":"https://openalex.org/I172675005","display_name":"Nanyang Technological University","ror":"https://ror.org/02e7b5302","country_code":"SG","type":"education","lineage":["https://openalex.org/I172675005"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Yang Liu","raw_affiliation_strings":["School of Computer Science and Engineering, Nanyang Technological University, Singapore, Singapore"],"affiliations":[{"raw_affiliation_string":"School of Computer Science and Engineering, Nanyang Technological University, Singapore, Singapore","institution_ids":["https://openalex.org/I172675005"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Pak-Lok Poon","orcid":"https://orcid.org/0000-0003-2840-2418"},"institutions":[{"id":"https://openalex.org/I74899385","display_name":"Central Queensland University","ror":"https://ror.org/023q4bk22","country_code":"AU","type":"education","lineage":["https://openalex.org/I74899385"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Pak-Lok Poon","raw_affiliation_strings":["School of Engineering and Technology, Central Queensland University, Melbourne, VIC, Australia"],"affiliations":[{"raw_affiliation_string":"School of Engineering and Technology, Central Queensland University, Melbourne, VIC, Australia","institution_ids":["https://openalex.org/I74899385"]}]},{"author_position":"last","author":{"id":null,"display_name":"Baowen Xu","orcid":"https://orcid.org/0000-0001-7743-1296"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Baowen Xu","raw_affiliation_strings":["State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China"],"affiliations":[{"raw_affiliation_string":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":6,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I37461747"],"apc_list":null,"apc_paid":null,"fwci":3.5655,"has_fulltext":true,"cited_by_count":59,"citation_normalized_percentile":{"value":0.939216,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":100},"biblio":{"volume":"69","issue":"4","first_page":"1293","last_page":"1322"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.1550000011920929,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.1550000011920929,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.11400000005960464,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13197","display_name":"Spreadsheets and End-User Computing","score":0.0737999975681305,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/unsupervised-learning","display_name":"Unsupervised learning","score":0.5946000218391418},{"id":"https://openalex.org/keywords/cluster-analysis","display_name":"Cluster analysis","score":0.49540001153945923},{"id":"https://openalex.org/keywords/computational-learning-theory","display_name":"Computational learning theory","score":0.4244999885559082},{"id":"https://openalex.org/keywords/a-priori-and-a-posteriori","display_name":"A priori and a posteriori","score":0.4043999910354614},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.3614000082015991},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.3474000096321106},{"id":"https://openalex.org/keywords/software-system","display_name":"Software system","score":0.3407000005245209}],"concepts":[{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.8705000281333923},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.8210999965667725},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7418000102043152},{"id":"https://openalex.org/C8038995","wikidata":"https://www.wikidata.org/wiki/Q1152135","display_name":"Unsupervised learning","level":2,"score":0.5946000218391418},{"id":"https://openalex.org/C73555534","wikidata":"https://www.wikidata.org/wiki/Q622825","display_name":"Cluster analysis","level":2,"score":0.49540001153945923},{"id":"https://openalex.org/C50292564","wikidata":"https://www.wikidata.org/wiki/Q2462783","display_name":"Computational learning theory","level":3,"score":0.4244999885559082},{"id":"https://openalex.org/C75553542","wikidata":"https://www.wikidata.org/wiki/Q178161","display_name":"A priori and a posteriori","level":2,"score":0.4043999910354614},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.3614000082015991},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.3474000096321106},{"id":"https://openalex.org/C149091818","wikidata":"https://www.wikidata.org/wiki/Q2429814","display_name":"Software system","level":3,"score":0.3407000005245209},{"id":"https://openalex.org/C115903097","wikidata":"https://www.wikidata.org/wiki/Q7094097","display_name":"Online machine learning","level":3,"score":0.32100000977516174},{"id":"https://openalex.org/C77967617","wikidata":"https://www.wikidata.org/wiki/Q4677561","display_name":"Active learning (machine learning)","level":2,"score":0.3181999921798706},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.3181999921798706},{"id":"https://openalex.org/C24138899","wikidata":"https://www.wikidata.org/wiki/Q17141258","display_name":"Instance-based learning","level":3,"score":0.2935999929904938},{"id":"https://openalex.org/C12267149","wikidata":"https://www.wikidata.org/wiki/Q282453","display_name":"Support vector machine","level":2,"score":0.28060001134872437},{"id":"https://openalex.org/C26713055","wikidata":"https://www.wikidata.org/wiki/Q245962","display_name":"Implementation","level":2,"score":0.27140000462532043},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.26260000467300415},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.26109999418258667},{"id":"https://openalex.org/C7166840","wikidata":"https://www.wikidata.org/wiki/Q1199682","display_name":"System testing","level":2,"score":0.2508000135421753}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1109/tr.2020.2972266","is_oa":true,"landing_page_url":"https://doi.org/10.1109/tr.2020.2972266","pdf_url":"https://ieeexplore.ieee.org/ielx7/24/9274445/09036058.pdf","source":{"id":"https://openalex.org/S87725633","display_name":"IEEE Transactions on Reliability","issn_l":"0018-9529","issn":["0018-9529","1558-1721"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Reliability","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:1807.10453","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1807.10453","pdf_url":"https://arxiv.org/pdf/1807.10453","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:acquire.cqu.edu.au:cqu:20738","is_oa":true,"landing_page_url":"http://hdl.cqu.edu.au/10018/1340432","pdf_url":null,"source":{"id":"https://openalex.org/S4306400635","display_name":"Acquire (CQUniversity)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I74899385","host_organization_name":"Central Queensland University","host_organization_lineage":["https://openalex.org/I74899385"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Xie, X, Zhang, Z, Chen, TY, Liu, Y, Poon, PL, Xu, B, (2020). METTLE: A METamorphic Testing approach to assessing and validating unsupervised machine LEarning systems. IEEE Transactions on Reliability, Vol. 69, No. 4,  p. 1293-1322 http://dx.doi.org/10.1109/TR.2020.2972266","raw_type":"journal article"},{"id":"pmh:oai:researchbank.swinburne.edu.au:05cac84c-0213-4b66-97e0-a77316a949f7/1","is_oa":false,"landing_page_url":"http://hdl.handle.net/1959.3/459336","pdf_url":null,"source":{"id":"https://openalex.org/S4306401157","display_name":"Swinburne Research Bank (Swinburne University of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I57093077","host_organization_name":"Swinburne University of Technology","host_organization_lineage":["https://openalex.org/I57093077"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Transactions on Reliability, Vol. 69, no. 4 (Dec 2020), pp. 1-30, 9036058","raw_type":""}],"best_oa_location":{"id":"doi:10.1109/tr.2020.2972266","is_oa":true,"landing_page_url":"https://doi.org/10.1109/tr.2020.2972266","pdf_url":"https://ieeexplore.ieee.org/ielx7/24/9274445/09036058.pdf","source":{"id":"https://openalex.org/S87725633","display_name":"IEEE Transactions on Reliability","issn_l":"0018-9529","issn":["0018-9529","1558-1721"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Reliability","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1023919524","display_name":null,"funder_award_id":", Grant","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G1121271761","display_name":null,"funder_award_id":"Program","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G1231421488","display_name":null,"funder_award_id":"under","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G1276726121","display_name":null,"funder_award_id":"61972289","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G1884936576","display_name":null,"funder_award_id":"61832009","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G2087396116","display_name":null,"funder_award_id":"China","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G3317480652","display_name":null,"funder_award_id":"Science","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G374735950","display_name":null,"funder_award_id":"61772263","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G391238517","display_name":null,"funder_award_id":", and","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5994120800","display_name":null,"funder_award_id":"Natural","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G8323144799","display_name":null,"funder_award_id":"61572375","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2951165326.pdf","grobid_xml":"https://content.openalex.org/works/W2951165326.grobid-xml"},"referenced_works_count":60,"referenced_works":["https://openalex.org/W1104960690","https://openalex.org/W1489991816","https://openalex.org/W1501760102","https://openalex.org/W1525129425","https://openalex.org/W1567073561","https://openalex.org/W1977556410","https://openalex.org/W1981367467","https://openalex.org/W1985059878","https://openalex.org/W2030765510","https://openalex.org/W2041650849","https://openalex.org/W2051526030","https://openalex.org/W2057923756","https://openalex.org/W2071177730","https://openalex.org/W2072240081","https://openalex.org/W2110379757","https://openalex.org/W2113586398","https://openalex.org/W2116219421","https://openalex.org/W2129018774","https://openalex.org/W2133517430","https://openalex.org/W2135900458","https://openalex.org/W2141012957","https://openalex.org/W2146598701","https://openalex.org/W2148108604","https://openalex.org/W2154897437","https://openalex.org/W2166392222","https://openalex.org/W2169649438","https://openalex.org/W2170111518","https://openalex.org/W2234763457","https://openalex.org/W2293505944","https://openalex.org/W2295673959","https://openalex.org/W2324595780","https://openalex.org/W2324662912","https://openalex.org/W2733722625","https://openalex.org/W2769245605","https://openalex.org/W2782311202","https://openalex.org/W2789057423","https://openalex.org/W2804337238","https://openalex.org/W2859484040","https://openalex.org/W2884652599","https://openalex.org/W2888307014","https://openalex.org/W2896505116","https://openalex.org/W2917321477","https://openalex.org/W2954629067","https://openalex.org/W2963327228","https://openalex.org/W2963905884","https://openalex.org/W2969812409","https://openalex.org/W2998768810","https://openalex.org/W4247878092","https://openalex.org/W4251827675","https://openalex.org/W4254332277","https://openalex.org/W4291213652","https://openalex.org/W6631148570","https://openalex.org/W6635035540","https://openalex.org/W6637131181","https://openalex.org/W6637231022","https://openalex.org/W6675354045","https://openalex.org/W6685179555","https://openalex.org/W6685380521","https://openalex.org/W6765411980","https://openalex.org/W6774112282"],"related_works":[],"abstract_inverted_index":{"Unsupervised":[0],"machine":[1,34,108,128,168,237],"learning":[2,35,129,169,238],"is":[3,14],"the":[4,24,45,80,87,122,134,183,233,236],"training":[5],"of":[6,47,82,86,91,126,139,187,232,235],"an":[7,174,241],"artificial":[8],"intelligence":[9],"system":[10],"using":[11],"information":[12],"that":[13,163],"neither":[15],"classified":[16],"nor":[17],"labeled,":[18],"with":[19,54,218],"a":[20,30,83,99,117,177,248,256],"view":[21,90],"to":[22,56,79,103,120,181,208],"modeling":[23],"underlying":[25],"structure":[26],"or":[27,250],"distribution":[28],"in":[29,40,94,216],"dataset.":[31],"Since":[32],"unsupervised":[33,107],"systems":[36,49,141,215,239],"are":[37,64,75,206],"widely":[38],"used":[39],"many":[41],"real-world":[42],"applications,":[43],"assessing":[44,104],"appropriateness":[46],"these":[48,140],"and":[50,60,71,105,137,176,185,191,211,230],"validating":[51,106],"their":[52,219],"implementations":[53],"respect":[55],"individual":[57,143],"users'":[58,144,159],"requirements":[59,138],"specific":[61,135,221],"application":[62],"scenarios/contexts":[63],"indisputably":[65],"two":[66],"important":[67],"tasks.":[68],"Such":[69],"assessments":[70],"validation":[72],"tasks,":[73],"however,":[74],"fairly":[76],"challenging":[77],"due":[78],"absence":[81],"priori":[84],"knowledge":[85],"data.":[88],"In":[89],"this":[92,95],"challenge,":[93],"article,":[96],"we":[97,149],"develop":[98],"METamorphic":[100],"Testing":[101],"approach":[102,115],"LEarning":[109],"systems,":[110,130],"abbreviated":[111],"as":[112],"mettle.":[113,188],"Our":[114,189,223],"provides":[116],"new":[118],"way":[119],"unveil":[121],"(possibly":[123],"latent)":[124],"characteristics":[125,162],"various":[127],"by":[131,167,199],"explicitly":[132],"considering":[133],"expectations":[136],"from":[142,240],"perspectives.":[145],"To":[146],"support":[147],"mettle,":[148],"have":[150,172,195],"further":[151],"formulated":[152],"11":[153],"generic":[154],"metamorphic":[155],"relations":[156],"(MRs),":[157],"covering":[158],"generally":[160],"expected":[161],"should":[164],"be":[165],"possessed":[166],"systems.":[170],"We":[171],"performed":[173],"experiment":[175,190],"user":[178,192],"evaluation":[179,193],"study":[180,194],"evaluate":[182],"viability":[184],"effectiveness":[186],"shown":[196],"that,":[197],"guided":[198],"user-defined":[200],"MR-based":[201],"adequacy":[202],"criteria,":[203],"end":[204],"users":[205],"able":[207],"assess,":[209],"validate,":[210],"select":[212],"appropriate":[213],"clustering":[214],"accordance":[217],"own":[220],"needs.":[222],"investigation":[224],"has":[225],"also":[226],"yielded":[227],"insightful":[228],"understanding":[229],"interpretation":[231],"behavior":[234],"end-user":[242],"software":[243],"engineering's":[244],"perspective,":[245,252],"rather":[246],"than":[247],"designer's":[249],"implementor's":[251],"who":[253],"normally":[254],"adopts":[255],"theoretical":[257],"approach.":[258]},"counts_by_year":[{"year":2025,"cited_by_count":17},{"year":2024,"cited_by_count":16},{"year":2023,"cited_by_count":10},{"year":2022,"cited_by_count":10},{"year":2021,"cited_by_count":5},{"year":2020,"cited_by_count":1}],"updated_date":"2026-03-20T23:20:44.827607","created_date":"2019-06-27T00:00:00"}