{"id":"https://openalex.org/W2886585694","doi":"https://doi.org/10.1145/3230977.3230999","title":"Who Tests the Testers?","display_name":"Who Tests the Testers?","publication_year":2018,"publication_date":"2018-08-08","ids":{"openalex":"https://openalex.org/W2886585694","doi":"https://doi.org/10.1145/3230977.3230999","mag":"2886585694"},"language":"en","primary_location":{"id":"doi:10.1145/3230977.3230999","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3230977.3230999","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2018 ACM Conference on International Computing Education Research","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5014228001","display_name":"John Wrenn","orcid":"https://orcid.org/0000-0002-0588-8649"},"institutions":[{"id":"https://openalex.org/I27804330","display_name":"Brown University","ror":"https://ror.org/05gq02987","country_code":"US","type":"education","lineage":["https://openalex.org/I27804330"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"John Wrenn","raw_affiliation_strings":["Brown University, Providence, RI, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Brown University, Providence, RI, USA","institution_ids":["https://openalex.org/I27804330"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024394821","display_name":"Shriram Krishnamurthi","orcid":"https://orcid.org/0000-0001-5184-1975"},"institutions":[{"id":"https://openalex.org/I27804330","display_name":"Brown University","ror":"https://ror.org/05gq02987","country_code":"US","type":"education","lineage":["https://openalex.org/I27804330"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shriram Krishnamurthi","raw_affiliation_strings":["Brown University, Providence, RI, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Brown University, Providence, RI, USA","institution_ids":["https://openalex.org/I27804330"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5047286412","display_name":"Kathi Fisler","orcid":"https://orcid.org/0000-0002-7895-8206"},"institutions":[{"id":"https://openalex.org/I27804330","display_name":"Brown University","ror":"https://ror.org/05gq02987","country_code":"US","type":"education","lineage":["https://openalex.org/I27804330"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kathi Fisler","raw_affiliation_strings":["Brown University, Providence, RI, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Brown University, Providence, RI, USA","institution_ids":["https://openalex.org/I27804330"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I27804330"],"apc_list":null,"apc_paid":null,"fwci":1.7486,"has_fulltext":false,"cited_by_count":24,"citation_normalized_percentile":{"value":0.85885886,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"51","last_page":"59"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.788561224937439},{"id":"https://openalex.org/keywords/variety","display_name":"Variety (cybernetics)","score":0.6817722916603088},{"id":"https://openalex.org/keywords/implementation","display_name":"Implementation","score":0.6538140773773193},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.5454831719398499},{"id":"https://openalex.org/keywords/point","display_name":"Point (geometry)","score":0.5287069082260132},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.5191951990127563},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.39784109592437744},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3950892686843872},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3869367241859436}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.788561224937439},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.6817722916603088},{"id":"https://openalex.org/C26713055","wikidata":"https://www.wikidata.org/wiki/Q245962","display_name":"Implementation","level":2,"score":0.6538140773773193},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.5454831719398499},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.5287069082260132},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.5191951990127563},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.39784109592437744},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3950892686843872},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3869367241859436},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3230977.3230999","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3230977.3230999","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2018 ACM Conference on International Computing Education Research","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.7400000095367432}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":34,"referenced_works":["https://openalex.org/W1540641082","https://openalex.org/W1597756446","https://openalex.org/W1967294945","https://openalex.org/W1967709752","https://openalex.org/W1967788581","https://openalex.org/W1971590752","https://openalex.org/W1971973232","https://openalex.org/W1981006850","https://openalex.org/W1982316994","https://openalex.org/W1983477653","https://openalex.org/W1988335552","https://openalex.org/W2005253577","https://openalex.org/W2006682633","https://openalex.org/W2030888081","https://openalex.org/W2032210625","https://openalex.org/W2060853440","https://openalex.org/W2078518223","https://openalex.org/W2084892841","https://openalex.org/W2085617435","https://openalex.org/W2092989624","https://openalex.org/W2095423622","https://openalex.org/W2101052930","https://openalex.org/W2106072155","https://openalex.org/W2106390619","https://openalex.org/W2119717320","https://openalex.org/W2137109811","https://openalex.org/W2137846723","https://openalex.org/W2144902156","https://openalex.org/W2165612380","https://openalex.org/W2173213060","https://openalex.org/W2408482380","https://openalex.org/W2732305299","https://openalex.org/W4244488020","https://openalex.org/W4247826686"],"related_works":["https://openalex.org/W2120447654","https://openalex.org/W2977179488","https://openalex.org/W2144453115","https://openalex.org/W2744553356","https://openalex.org/W2128223750","https://openalex.org/W4238532390","https://openalex.org/W2188872161","https://openalex.org/W2961779879","https://openalex.org/W797688974","https://openalex.org/W3217486089"],"abstract_inverted_index":{"Instructors":[0],"routinely":[1],"use":[2],"automated":[3,26,111],"assessment":[4,27,37,42],"methods":[5,28],"to":[6,33,49],"evaluate":[7],"the":[8,30,58,61,94,108],"semantic":[9],"qualities":[10],"of":[11,25,36,60,71,96,110],"student":[12],"implementations":[13],"and,":[14],"sometimes,":[15],"test":[16],"suites.":[17],"In":[18],"this":[19],"work,":[20],"we":[21,64],"distill":[22],"a":[23,34],"variety":[24],"in":[29,44,68],"literature":[31],"down":[32],"pair":[35],"models.":[38],"We":[39,76,98],"identify":[40],"pathological":[41],"outcomes":[43],"each":[45],"model":[46],"that":[47,79,90,100],"point":[48],"underlying":[50],"methodological":[51],"flaws.":[52],"These":[53],"theoretical":[54],"flaws":[55,82],"broadly":[56],"threaten":[57],"validity":[59],"techniques,":[62],"and":[63,83],"actually":[65],"observe":[66],"them":[67],"multiple":[69],"assignments":[70],"an":[72],"introductory":[73],"programming":[74],"course.":[75],"propose":[77],"adjustments":[78],"remedy":[80],"these":[81,87,102],"then":[84],"demonstrate,":[85],"on":[86],"same":[88],"assignments,":[89],"our":[91],"interventions":[92],"improve":[93,107],"accuracy":[95,109],"assessment.":[97,112],"believe":[99],"with":[101],"adjustments,":[103],"instructors":[104],"can":[105],"greatly":[106]},"counts_by_year":[{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":7},{"year":2022,"cited_by_count":4},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":3},{"year":2019,"cited_by_count":2}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}