{"id":"https://openalex.org/W3178012978","doi":"https://doi.org/10.1145/3470652","title":"A Systematic Literature Review of Empiricism and Norms of Reporting in Computing Education Research Literature","display_name":"A Systematic Literature Review of Empiricism and Norms of Reporting in Computing Education Research Literature","publication_year":2021,"publication_date":"2021-10-18","ids":{"openalex":"https://openalex.org/W3178012978","doi":"https://doi.org/10.1145/3470652","mag":"3178012978"},"language":"en","primary_location":{"id":"doi:10.1145/3470652","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3470652","pdf_url":null,"source":{"id":"https://openalex.org/S2764465379","display_name":"ACM Transactions on Computing Education","issn_l":"1946-6226","issn":["1946-6226"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Computing Education","raw_type":"journal-article"},"type":"preprint","indexed_in":["crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"http://arxiv.org/pdf/2107.01984.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5013416767","display_name":"Sarah Heckman","orcid":"https://orcid.org/0000-0003-4351-8611"},"institutions":[{"id":"https://openalex.org/I137902535","display_name":"North Carolina State University","ror":"https://ror.org/04tj63d06","country_code":"US","type":"education","lineage":["https://openalex.org/I137902535"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Sarah Heckman","raw_affiliation_strings":["North Carolina State University, Campus Box, Raleigh"],"affiliations":[{"raw_affiliation_string":"North Carolina State University, Campus Box, Raleigh","institution_ids":["https://openalex.org/I137902535"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005826696","display_name":"Jeffrey C. Carver","orcid":"https://orcid.org/0000-0002-7824-9151"},"institutions":[{"id":"https://openalex.org/I17301866","display_name":"University of Alabama","ror":"https://ror.org/03xrrjk67","country_code":"US","type":"education","lineage":["https://openalex.org/I17301866"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jeffrey C. Carver","raw_affiliation_strings":["University of Alabama, Tuscaloosa, AL"],"affiliations":[{"raw_affiliation_string":"University of Alabama, Tuscaloosa, AL","institution_ids":["https://openalex.org/I17301866"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036558021","display_name":"Mark Sherriff","orcid":"https://orcid.org/0000-0002-1745-205X"},"institutions":[{"id":"https://openalex.org/I51556381","display_name":"University of Virginia","ror":"https://ror.org/0153tk833","country_code":"US","type":"education","lineage":["https://openalex.org/I51556381"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mark Sherriff","raw_affiliation_strings":["University of Virginia, Charlottesville, VA"],"affiliations":[{"raw_affiliation_string":"University of Virginia, Charlottesville, VA","institution_ids":["https://openalex.org/I51556381"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5048570710","display_name":"Ahmed Al-Zubidy","orcid":"https://orcid.org/0000-0003-4305-2153"},"institutions":[{"id":"https://openalex.org/I17301866","display_name":"University of Alabama","ror":"https://ror.org/03xrrjk67","country_code":"US","type":"education","lineage":["https://openalex.org/I17301866"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ahmed Al-zubidy","raw_affiliation_strings":["University of Alabama, Tuscaloosa, AL"],"affiliations":[{"raw_affiliation_string":"University of Alabama, Tuscaloosa, AL","institution_ids":["https://openalex.org/I17301866"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5013416767"],"corresponding_institution_ids":["https://openalex.org/I137902535"],"apc_list":null,"apc_paid":null,"fwci":0.7292,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.76664232,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":96,"max":97},"biblio":{"volume":"22","issue":"1","first_page":"1","last_page":"46"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.9947999715805054,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.9947999715805054,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.9847000241279602,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.9829999804496765,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/rubric","display_name":"Rubric","score":0.6379669904708862},{"id":"https://openalex.org/keywords/systematic-review","display_name":"Systematic review","score":0.5978319048881531},{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.5459160208702087},{"id":"https://openalex.org/keywords/psychological-intervention","display_name":"Psychological intervention","score":0.5121371746063232},{"id":"https://openalex.org/keywords/curriculum","display_name":"Curriculum","score":0.4860716164112091},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.46323081851005554},{"id":"https://openalex.org/keywords/empiricism","display_name":"Empiricism","score":0.4434365928173065},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.4216502010822296},{"id":"https://openalex.org/keywords/inclusion","display_name":"Inclusion (mineral)","score":0.4191027283668518},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.40781837701797485},{"id":"https://openalex.org/keywords/mathematics-education","display_name":"Mathematics education","score":0.23583906888961792},{"id":"https://openalex.org/keywords/pedagogy","display_name":"Pedagogy","score":0.16290616989135742},{"id":"https://openalex.org/keywords/medline","display_name":"MEDLINE","score":0.15516093373298645},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.14202430844306946},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.13241207599639893},{"id":"https://openalex.org/keywords/social-psychology","display_name":"Social psychology","score":0.11117646098136902}],"concepts":[{"id":"https://openalex.org/C111640148","wikidata":"https://www.wikidata.org/wiki/Q847349","display_name":"Rubric","level":2,"score":0.6379669904708862},{"id":"https://openalex.org/C189708586","wikidata":"https://www.wikidata.org/wiki/Q1504425","display_name":"Systematic review","level":3,"score":0.5978319048881531},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.5459160208702087},{"id":"https://openalex.org/C27415008","wikidata":"https://www.wikidata.org/wiki/Q7256382","display_name":"Psychological intervention","level":2,"score":0.5121371746063232},{"id":"https://openalex.org/C47177190","wikidata":"https://www.wikidata.org/wiki/Q207137","display_name":"Curriculum","level":2,"score":0.4860716164112091},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.46323081851005554},{"id":"https://openalex.org/C36790819","wikidata":"https://www.wikidata.org/wiki/Q83368","display_name":"Empiricism","level":2,"score":0.4434365928173065},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.4216502010822296},{"id":"https://openalex.org/C109359841","wikidata":"https://www.wikidata.org/wiki/Q728944","display_name":"Inclusion (mineral)","level":2,"score":0.4191027283668518},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.40781837701797485},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.23583906888961792},{"id":"https://openalex.org/C19417346","wikidata":"https://www.wikidata.org/wiki/Q7922","display_name":"Pedagogy","level":1,"score":0.16290616989135742},{"id":"https://openalex.org/C2779473830","wikidata":"https://www.wikidata.org/wiki/Q1540899","display_name":"MEDLINE","level":2,"score":0.15516093373298645},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.14202430844306946},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.13241207599639893},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.11117646098136902},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0},{"id":"https://openalex.org/C118552586","wikidata":"https://www.wikidata.org/wiki/Q7867","display_name":"Psychiatry","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3470652","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3470652","pdf_url":null,"source":{"id":"https://openalex.org/S2764465379","display_name":"ACM Transactions on Computing Education","issn_l":"1946-6226","issn":["1946-6226"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Computing Education","raw_type":"journal-article"},{"id":"mag:3178012978","is_oa":true,"landing_page_url":"http://arxiv.org/pdf/2107.01984.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"doi:10.48550/arxiv.2107.01984","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2107.01984","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article-journal"}],"best_oa_location":{"id":"mag:3178012978","is_oa":true,"landing_page_url":"http://arxiv.org/pdf/2107.01984.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},"sustainable_development_goals":[{"display_name":"Quality Education","score":0.7699999809265137,"id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":67,"referenced_works":["https://openalex.org/W1587555836","https://openalex.org/W1876404477","https://openalex.org/W1881792686","https://openalex.org/W1966463724","https://openalex.org/W1971621932","https://openalex.org/W2000851492","https://openalex.org/W2000911559","https://openalex.org/W2004740500","https://openalex.org/W2009659869","https://openalex.org/W2010333176","https://openalex.org/W2021458971","https://openalex.org/W2026222769","https://openalex.org/W2036586008","https://openalex.org/W2056004316","https://openalex.org/W2065994439","https://openalex.org/W2081312500","https://openalex.org/W2090471876","https://openalex.org/W2108696783","https://openalex.org/W2113085067","https://openalex.org/W2125710299","https://openalex.org/W2130671839","https://openalex.org/W2153523756","https://openalex.org/W2162739315","https://openalex.org/W2167042818","https://openalex.org/W2168894761","https://openalex.org/W2170274526","https://openalex.org/W2264893711","https://openalex.org/W2284289670","https://openalex.org/W2290724249","https://openalex.org/W2481029616","https://openalex.org/W2513097092","https://openalex.org/W2549453003","https://openalex.org/W2626666917","https://openalex.org/W2742031688","https://openalex.org/W2790611565","https://openalex.org/W2887345162","https://openalex.org/W2889140186","https://openalex.org/W2896471517","https://openalex.org/W2910880405","https://openalex.org/W2910926550","https://openalex.org/W2915075304","https://openalex.org/W2916376035","https://openalex.org/W2916826581","https://openalex.org/W2917832570","https://openalex.org/W2921175316","https://openalex.org/W2942575417","https://openalex.org/W2964513566","https://openalex.org/W2966268134","https://openalex.org/W2969260140","https://openalex.org/W2995281728","https://openalex.org/W2996454454","https://openalex.org/W3000021527","https://openalex.org/W3003033758","https://openalex.org/W3015083106","https://openalex.org/W3023711780","https://openalex.org/W3033689366","https://openalex.org/W3047718235","https://openalex.org/W3047801365","https://openalex.org/W3048016185","https://openalex.org/W3048168536","https://openalex.org/W4205785435","https://openalex.org/W4210532326","https://openalex.org/W4231474584","https://openalex.org/W4251662847","https://openalex.org/W4253609317","https://openalex.org/W4394428966","https://openalex.org/W6684453849"],"related_works":["https://openalex.org/W3132605238","https://openalex.org/W3004986056","https://openalex.org/W3161889565","https://openalex.org/W1980420734","https://openalex.org/W3202639482","https://openalex.org/W2091793466","https://openalex.org/W2615383388","https://openalex.org/W2042939403","https://openalex.org/W3206149695","https://openalex.org/W2142134161","https://openalex.org/W2342057325","https://openalex.org/W3119755789","https://openalex.org/W3001961307","https://openalex.org/W2047846647","https://openalex.org/W2046283373","https://openalex.org/W3165424749","https://openalex.org/W2551923089","https://openalex.org/W2057318732","https://openalex.org/W2939916325","https://openalex.org/W2893669639"],"abstract_inverted_index":{"Context.":[0],"Computing":[1,94,214,231],"Education":[2,95,208,215,226,232,237],"Research":[3,96,216],"(CER)":[4],"is":[5,61,86],"critical":[6],"to":[7,24,34,46,52,64,73,87,107,120,248,282,303,407,426,440,471,477],"help":[8,472],"the":[9,17,41,70,89,139,147,150,155,192,243,249,268,273,279,284,291,328,373,379,416,427,452,479],"computing":[10,26,474,483],"education":[11,475],"community":[12,33,72,463],"and":[13,57,77,111,168,194,221,234,241,253,260,361,381,405,455,459],"policy":[14],"makers":[15],"support":[16,478],"increasing":[18],"population":[19],"of":[20,83,91,126,134,149,171,191,199,286,311,321,356,399,415,467,482],"students":[21],"who":[22],"need":[23,63],"learn":[25],"skills":[27],"for":[28,105,166,169,174,180,271,290,375,433,457],"future":[29],"careers.":[30],"For":[31],"a":[32,39,62,187,348],"systematically":[35,74],"advance":[36,75],"knowledge":[37,76],"about":[38,445],"topic,":[40],"members":[42],"must":[43],"be":[44],"able":[45],"understand":[47,65],"published":[48,67,254],"work":[49,447],"thoroughly":[50],"enough":[51],"perform":[53,108],"replications,":[54,109],"conduct":[55],"meta-analyses,":[56,110],"build":[58,78],"theories.":[59,79],"There":[60,346],"whether":[66,100],"research":[68,117,391,394],"allows":[69],"CER":[71,129,201,244,382,417,420,468],"Objectives.":[80],"The":[81],"goal":[82],"this":[84,121],"study":[85,401,453],"characterize":[88,283],"reporting":[90,181,434,466],"empiricism":[92],"in":[93,128,223,350,378],"literature":[97,189],"by":[98],"identifying":[99],"publications":[101],"include":[102],"content":[103],"necessary":[104],"researchers":[106],"theory":[112,476],"building.":[113],"We":[114,185,239,306,437],"answer":[115],"three":[116],"questions":[118,297],"related":[119,370],"goal:":[122],"(RQ1)":[123],"What":[124],"percentage":[125],"papers":[127,140,156,251,312,351,368,386],"venues":[130,257,317],"have":[131,142,158],"some":[132,319,354,362],"form":[133,320],"empirical":[135,143,151,159,182,322,424],"evaluation?":[136,152],"(RQ2)":[137],"Of":[138,154],"that":[141,157,308,352],"evaluation,":[144,160],"what":[145],"are":[146,422,435],"characteristics":[148],"(RQ3)":[153],"do":[161],"they":[162],"follow":[163],"norms":[164,285,432],"(both":[165],"inclusion":[167],"labeling":[170],"information":[172],"needed":[173],"replication,":[175],"meta-analysis,":[176],"and,":[177],"eventually,":[178],"theory-building)":[179],"work?":[183],"Methods.":[184],"conducted":[186],"systematic":[188],"review":[190],"2014":[193,259],"2015":[195],"proceedings":[196],"or":[197,296,344,365,396],"issues":[198],"five":[200,316],"venues:":[202],"Technical":[203],"Symposium":[204,212],"on":[205,213,219,230,337],"Computer":[206,224,235],"Science":[207,225,236],"(SIGCSE":[209],"TS),":[210],"International":[211],"(ICER),":[217],"Conference":[218],"Innovation":[220],"Technology":[222],"(ITiCSE),":[227],"ACM":[228],"Transactions":[229],"(TOCE),":[233],"(CSE).":[238],"developed":[240],"applied":[242,278],"Empiricism":[245],"Assessment":[246],"Rubric":[247,270],"427":[250],"accepted":[252,313],"at":[255],"these":[256],"over":[258,309],"2015.":[261],"Two":[262],"people":[263],"evaluated":[264],"each":[265],"paper":[266,292],"using":[267],"Base":[269],"characterizing":[272],"paper.":[274],"An":[275],"individual":[276],"person":[277],"other":[280,363],"rubrics":[281],"reporting,":[287],"as":[288],"appropriate":[289],"type.":[293],"Any":[294],"discrepancies":[295],"were":[298,327,387],"discussed":[299],"between":[300,358],"multiple":[301],"reviewers":[302],"resolve.":[304],"Results.":[305],"found":[307],"80%":[310],"across":[314],"all":[315,431],"had":[318,353],"evaluation.":[323],"Quantitative":[324],"evaluation":[325],"methods":[326],"most":[329,333],"frequently":[330,334],"reported.":[331],"Papers":[332],"reported":[335,369,390],"results":[336,410,425,456],"interventions":[338],"around":[339],"pedagogical":[340],"techniques,":[341],"curriculum,":[342],"community,":[343],"tools.":[345],"was":[347],"split":[349],"type":[355],"comparison":[357],"an":[359],"intervention":[360],"dataset":[364],"baseline.":[366],"Most":[367],"work,":[371],"following":[372],"expectations":[374],"doing":[376],"so":[377,448],"SIGCSE":[380],"community.":[383],"However,":[384],"many":[385],"lacking":[388],"properly":[389],"objectives,":[392],"goals,":[393],"questions,":[395],"hypotheses;":[397],"description":[398],"participants;":[400],"design;":[402],"data":[403],"collection;":[404],"threats":[406],"validity.":[408],"These":[409],"align":[411],"with":[412],"prior":[413],"surveys":[414],"literature.":[418],"Conclusions.":[419],"authors":[421,439],"contributing":[423],"literature;":[428],"however,":[429],"not":[430],"met.":[436],"encourage":[438],"provide":[441],"clear,":[442],"labeled":[443],"details":[444],"their":[446],"readers":[449],"can":[450],"use":[451],"methodologies":[454],"replications":[458],"meta-analyses.":[460],"As":[461],"our":[462,465],"grows,":[464],"should":[469],"mature":[470],"establish":[473],"next":[480],"generation":[481],"learners.":[484]},"counts_by_year":[{"year":2022,"cited_by_count":3}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}