{"id":"https://openalex.org/W3010192453","doi":"https://doi.org/10.1145/3328778.3366879","title":"A Validated Scoring Rubric for Explain-in-Plain-English Questions","display_name":"A Validated Scoring Rubric for Explain-in-Plain-English Questions","publication_year":2020,"publication_date":"2020-02-25","ids":{"openalex":"https://openalex.org/W3010192453","doi":"https://doi.org/10.1145/3328778.3366879","mag":"3010192453"},"language":"en","primary_location":{"id":"doi:10.1145/3328778.3366879","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3328778.3366879","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3328778.3366879","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 51st ACM Technical Symposium on Computer Science Education","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3328778.3366879","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101837457","display_name":"Binglin Chen","orcid":"https://orcid.org/0000-0001-9033-1281"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Binglin Chen","raw_affiliation_strings":["University of Illinois at Urbana-Champaign, Urbana, IL, USA"],"affiliations":[{"raw_affiliation_string":"University of Illinois at Urbana-Champaign, Urbana, IL, USA","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5077275818","display_name":"Sushmita Azad","orcid":"https://orcid.org/0000-0001-9239-9255"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sushmita Azad","raw_affiliation_strings":["University of Illinois at Urbana-Champaign, Urbana, IL, USA"],"affiliations":[{"raw_affiliation_string":"University of Illinois at Urbana-Champaign, Urbana, IL, USA","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034389713","display_name":"Rajarshi Haldar","orcid":null},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Rajarshi Haldar","raw_affiliation_strings":["University of Illinois at Urbana-Champaign, Urbana, IL, USA"],"affiliations":[{"raw_affiliation_string":"University of Illinois at Urbana-Champaign, Urbana, IL, USA","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001110474","display_name":"Matthew West","orcid":"https://orcid.org/0000-0002-7605-0050"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Matthew West","raw_affiliation_strings":["University of Illinois at Urbana-Champaign, Urbana, IL, USA"],"affiliations":[{"raw_affiliation_string":"University of Illinois at Urbana-Champaign, Urbana, IL, USA","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5066756473","display_name":"Craig Zilles","orcid":"https://orcid.org/0000-0003-4601-4398"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Craig Zilles","raw_affiliation_strings":["University of Illinois at Urbana-Champaign, Urbana, IL, USA"],"affiliations":[{"raw_affiliation_string":"University of Illinois at Urbana-Champaign, Urbana, IL, USA","institution_ids":["https://openalex.org/I157725225"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5101837457"],"corresponding_institution_ids":["https://openalex.org/I157725225"],"apc_list":null,"apc_paid":null,"fwci":4.5374,"has_fulltext":true,"cited_by_count":30,"citation_normalized_percentile":{"value":0.95153368,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"563","last_page":"569"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9969000220298767,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9969000220298767,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.9965000152587891,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9708999991416931,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/rubric","display_name":"Rubric","score":0.9340736865997314},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7285003662109375},{"id":"https://openalex.org/keywords/cronbachs-alpha","display_name":"Cronbach's alpha","score":0.7098202705383301},{"id":"https://openalex.org/keywords/reading","display_name":"Reading (process)","score":0.5885847806930542},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.5784825682640076},{"id":"https://openalex.org/keywords/correctness","display_name":"Correctness","score":0.568817675113678},{"id":"https://openalex.org/keywords/ambiguity","display_name":"Ambiguity","score":0.5261135697364807},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.4769517481327057},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.4507014751434326},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.4454360008239746},{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.4328545331954956},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.36154991388320923},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.34700101613998413},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.32596004009246826},{"id":"https://openalex.org/keywords/mathematics-education","display_name":"Mathematics education","score":0.2590615749359131},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.2042008340358734},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.19534063339233398},{"id":"https://openalex.org/keywords/psychometrics","display_name":"Psychometrics","score":0.12068948149681091}],"concepts":[{"id":"https://openalex.org/C111640148","wikidata":"https://www.wikidata.org/wiki/Q847349","display_name":"Rubric","level":2,"score":0.9340736865997314},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7285003662109375},{"id":"https://openalex.org/C106906290","wikidata":"https://www.wikidata.org/wiki/Q1140909","display_name":"Cronbach's alpha","level":3,"score":0.7098202705383301},{"id":"https://openalex.org/C554936623","wikidata":"https://www.wikidata.org/wiki/Q199657","display_name":"Reading (process)","level":2,"score":0.5885847806930542},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5784825682640076},{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.568817675113678},{"id":"https://openalex.org/C2780522230","wikidata":"https://www.wikidata.org/wiki/Q1140419","display_name":"Ambiguity","level":2,"score":0.5261135697364807},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4769517481327057},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.4507014751434326},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.4454360008239746},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.4328545331954956},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.36154991388320923},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.34700101613998413},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.32596004009246826},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.2590615749359131},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.2042008340358734},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.19534063339233398},{"id":"https://openalex.org/C171606756","wikidata":"https://www.wikidata.org/wiki/Q506132","display_name":"Psychometrics","level":2,"score":0.12068948149681091},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C70410870","wikidata":"https://www.wikidata.org/wiki/Q199906","display_name":"Clinical psychology","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3328778.3366879","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3328778.3366879","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3328778.3366879","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 51st ACM Technical Symposium on Computer Science Education","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3328778.3366879","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3328778.3366879","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3328778.3366879","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 51st ACM Technical Symposium on Computer Science Education","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.8999999761581421}],"awards":[{"id":"https://openalex.org/G183946574","display_name":null,"funder_award_id":"1347722, 1150490","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3010192453.pdf","grobid_xml":"https://content.openalex.org/works/W3010192453.grobid-xml"},"referenced_works_count":34,"referenced_works":["https://openalex.org/W598355015","https://openalex.org/W1594737854","https://openalex.org/W1973719497","https://openalex.org/W1983193319","https://openalex.org/W1987790173","https://openalex.org/W1997509089","https://openalex.org/W2012560454","https://openalex.org/W2021698654","https://openalex.org/W2023754385","https://openalex.org/W2029974693","https://openalex.org/W2031681640","https://openalex.org/W2041103998","https://openalex.org/W2061935394","https://openalex.org/W2067438752","https://openalex.org/W2071792242","https://openalex.org/W2093055647","https://openalex.org/W2096036274","https://openalex.org/W2104695310","https://openalex.org/W2122166962","https://openalex.org/W2126428499","https://openalex.org/W2129741299","https://openalex.org/W2141929780","https://openalex.org/W2149941633","https://openalex.org/W2153222072","https://openalex.org/W2165365113","https://openalex.org/W2167759247","https://openalex.org/W2347061999","https://openalex.org/W2748071319","https://openalex.org/W2907766211","https://openalex.org/W2912010641","https://openalex.org/W2921544693","https://openalex.org/W2945621931","https://openalex.org/W4210764005","https://openalex.org/W4231748357"],"related_works":["https://openalex.org/W2527177243","https://openalex.org/W1987813225","https://openalex.org/W4379053243","https://openalex.org/W3021020626","https://openalex.org/W2955201676","https://openalex.org/W2153005530","https://openalex.org/W2330342969","https://openalex.org/W2777782248","https://openalex.org/W4213362990","https://openalex.org/W3087250444"],"abstract_inverted_index":{"Previous":[0],"research":[1],"has":[2],"identified":[3],"the":[4,48,67,82,127,151],"ability":[5],"to":[6,22,44,75,109,149,203,207],"read":[7],"code":[8,32,53,165,184,204],"and":[9,92,115],"understand":[10],"its":[11],"high-level":[12],"purpose":[13],"as":[14,211],"an":[15,147],"important":[16],"developmental":[17],"skill":[18],"that":[19,102,126,159,180],"is":[20],"harder":[21],"do":[23,45],"(for":[24],"a":[25,37,62,99,135,160,169,188,208],"given":[26,38],"piece":[27],"of":[28,85,90,141,153,163],"code)":[29],"than":[30,46],"executing":[31],"in":[33,56,111,187,213],"one's":[34],"head":[35],"for":[36,105,183],"input":[39],"(\"code":[40],"tracing\"),":[41],"but":[42,70],"easier":[43],"writing":[47,205],"code.":[49],"Prior":[50],"work":[51],"involving":[52],"reading":[54,166,185],"(\"Explain":[55],"plain":[57,112],"English\")":[58],"problems,":[59],"have":[60],"used":[61],"scoring":[63,106],"rubric":[64,101,119],"inspired":[65],"by":[66],"SOLO":[68],"taxonomy,":[69],"we":[71,97,103,116,124,144,157,178],"found":[72,212],"it":[73,78],"difficult":[74],"employ":[76],"because":[77],"didn't":[79],"adequately":[80],"handle":[81],"three":[83],"dimensions":[84],"answer":[86],"quality:":[87],"correctness,":[88],"level":[89],"abstraction,":[91],"ambiguity.":[93],"In":[94],"this":[95,118],"paper,":[96],"describe":[98],"7-point":[100],"developed":[104],"student":[107],"responses":[108],"\"Explain":[110],"English''":[113],"questions,":[114],"validate":[117],"through":[120],"four":[121],"means.":[122],"First,":[123],"find":[125,158,179],"scale":[128],"can":[129],"be":[130],"reliably":[131],"applied":[132],"with":[133,134],"median":[136],"Krippendorff's":[137],"alpha":[138,174],"(inter-rater":[139],"reliability)":[140],"0.775.":[142],"Second,":[143],"report":[145],"on":[146],"experiment":[148],"assess":[150],"validity":[152],"our":[154,181],"scale.":[155],"Third,":[156],"survey":[161],"consisting":[162],"12":[164],"questions":[167,186],"had":[168],"high":[170],"internal":[171],"consistency":[172],"(Cronbach's":[173],"=":[175,192,201],"0.954).":[176],"Last,":[177],"scores":[182],"large":[189],"enrollment":[190],"(N":[191],"452)":[193],"data":[194],"structures":[195],"course":[196],"are":[197],"correlated":[198],"(Pearson's":[199],"R":[200],"0.555)":[202],"performance":[206],"similar":[209],"degree":[210],"previous":[214],"work.":[215]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":6},{"year":2024,"cited_by_count":6},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":8},{"year":2021,"cited_by_count":6},{"year":2020,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}