{"id":"https://openalex.org/W3133741462","doi":"https://doi.org/10.1145/3408877.3432411","title":"An Empirical Study to Determine if Mutants Can Effectively Simulate Students' Programming Mistakes to Increase Tutors' Confidence in Autograding","display_name":"An Empirical Study to Determine if Mutants Can Effectively Simulate Students' Programming Mistakes to Increase Tutors' Confidence in Autograding","publication_year":2021,"publication_date":"2021-03-03","ids":{"openalex":"https://openalex.org/W3133741462","doi":"https://doi.org/10.1145/3408877.3432411","mag":"3133741462"},"language":"en","primary_location":{"id":"doi:10.1145/3408877.3432411","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3408877.3432411","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 52nd ACM Technical Symposium on Computer Science Education","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5028238692","display_name":"Benjamin Clegg","orcid":"https://orcid.org/0000-0002-1323-7133"},"institutions":[{"id":"https://openalex.org/I91136226","display_name":"University of Sheffield","ror":"https://ror.org/05krs5044","country_code":"GB","type":"education","lineage":["https://openalex.org/I91136226"]}],"countries":["GB"],"is_corresponding":true,"raw_author_name":"Benjamin Simon Clegg","raw_affiliation_strings":["University of Sheffield, Sheffield, United Kingdom"],"affiliations":[{"raw_affiliation_string":"University of Sheffield, Sheffield, United Kingdom","institution_ids":["https://openalex.org/I91136226"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078598020","display_name":"Phil McMinn","orcid":"https://orcid.org/0000-0001-9137-7433"},"institutions":[{"id":"https://openalex.org/I91136226","display_name":"University of Sheffield","ror":"https://ror.org/05krs5044","country_code":"GB","type":"education","lineage":["https://openalex.org/I91136226"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Phil McMinn","raw_affiliation_strings":["University of Sheffield, Sheffield, United Kingdom"],"affiliations":[{"raw_affiliation_string":"University of Sheffield, Sheffield, United Kingdom","institution_ids":["https://openalex.org/I91136226"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5079261847","display_name":"Gordon Fraser","orcid":"https://orcid.org/0000-0002-4364-6595"},"institutions":[{"id":"https://openalex.org/I186354981","display_name":"University of Passau","ror":"https://ror.org/05ydjnb78","country_code":"DE","type":"education","lineage":["https://openalex.org/I186354981"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Gordon Fraser","raw_affiliation_strings":["University of Passau, Passau, Germany"],"affiliations":[{"raw_affiliation_string":"University of Passau, Passau, Germany","institution_ids":["https://openalex.org/I186354981"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5028238692"],"corresponding_institution_ids":["https://openalex.org/I91136226"],"apc_list":null,"apc_paid":null,"fwci":0.905,"has_fulltext":false,"cited_by_count":6,"citation_normalized_percentile":{"value":0.73218673,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"1055","last_page":"1061"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.995199978351593,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9951000213623047,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/grading","display_name":"Grading (engineering)","score":0.6976370811462402},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6843882203102112},{"id":"https://openalex.org/keywords/limiting","display_name":"Limiting","score":0.6187093257904053},{"id":"https://openalex.org/keywords/java","display_name":"Java","score":0.5851953625679016},{"id":"https://openalex.org/keywords/mutant","display_name":"Mutant","score":0.5772307515144348},{"id":"https://openalex.org/keywords/software-testing","display_name":"Software testing","score":0.49427247047424316},{"id":"https://openalex.org/keywords/mutation-testing","display_name":"Mutation testing","score":0.45742806792259216},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.44857725501060486},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.43914899230003357},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.4388110339641571},{"id":"https://openalex.org/keywords/test-case","display_name":"Test case","score":0.41628628969192505},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.3842133581638336},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3505043089389801},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.338978111743927},{"id":"https://openalex.org/keywords/mutation","display_name":"Mutation","score":0.28555208444595337},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.17896974086761475},{"id":"https://openalex.org/keywords/biology","display_name":"Biology","score":0.10176187753677368},{"id":"https://openalex.org/keywords/regression-analysis","display_name":"Regression analysis","score":0.09004455804824829}],"concepts":[{"id":"https://openalex.org/C2777286243","wikidata":"https://www.wikidata.org/wiki/Q5591926","display_name":"Grading (engineering)","level":2,"score":0.6976370811462402},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6843882203102112},{"id":"https://openalex.org/C188198153","wikidata":"https://www.wikidata.org/wiki/Q1613840","display_name":"Limiting","level":2,"score":0.6187093257904053},{"id":"https://openalex.org/C548217200","wikidata":"https://www.wikidata.org/wiki/Q251","display_name":"Java","level":2,"score":0.5851953625679016},{"id":"https://openalex.org/C143065580","wikidata":"https://www.wikidata.org/wiki/Q3285695","display_name":"Mutant","level":3,"score":0.5772307515144348},{"id":"https://openalex.org/C2984328558","wikidata":"https://www.wikidata.org/wiki/Q188522","display_name":"Software testing","level":3,"score":0.49427247047424316},{"id":"https://openalex.org/C163565370","wikidata":"https://www.wikidata.org/wiki/Q4308623","display_name":"Mutation testing","level":4,"score":0.45742806792259216},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.44857725501060486},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.43914899230003357},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.4388110339641571},{"id":"https://openalex.org/C128942645","wikidata":"https://www.wikidata.org/wiki/Q1568346","display_name":"Test case","level":3,"score":0.41628628969192505},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.3842133581638336},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3505043089389801},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.338978111743927},{"id":"https://openalex.org/C501734568","wikidata":"https://www.wikidata.org/wiki/Q42918","display_name":"Mutation","level":3,"score":0.28555208444595337},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.17896974086761475},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.10176187753677368},{"id":"https://openalex.org/C152877465","wikidata":"https://www.wikidata.org/wiki/Q208042","display_name":"Regression analysis","level":2,"score":0.09004455804824829},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C78519656","wikidata":"https://www.wikidata.org/wiki/Q101333","display_name":"Mechanical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C147176958","wikidata":"https://www.wikidata.org/wiki/Q77590","display_name":"Civil engineering","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3408877.3432411","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3408877.3432411","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 52nd ACM Technical Symposium on Computer Science Education","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.6800000071525574}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":18,"referenced_works":["https://openalex.org/W1950030762","https://openalex.org/W1971650562","https://openalex.org/W2012312630","https://openalex.org/W2028990673","https://openalex.org/W2029517828","https://openalex.org/W2049695835","https://openalex.org/W2075445701","https://openalex.org/W2135841285","https://openalex.org/W2135937266","https://openalex.org/W2241046636","https://openalex.org/W2399318684","https://openalex.org/W2474318526","https://openalex.org/W2616479610","https://openalex.org/W2682664750","https://openalex.org/W2789235734","https://openalex.org/W2792099437","https://openalex.org/W3009809749","https://openalex.org/W3111827872"],"related_works":["https://openalex.org/W2137109811","https://openalex.org/W2051971294","https://openalex.org/W2058234333","https://openalex.org/W17934386","https://openalex.org/W2780362549","https://openalex.org/W2008675462","https://openalex.org/W2135841285","https://openalex.org/W2159249596","https://openalex.org/W4256638442","https://openalex.org/W4289931172"],"abstract_inverted_index":{"Automated":[0],"grading":[1,214],"often":[2],"requires":[3],"automated":[4],"test":[5,78,192,215],"suites":[6],"to":[7,76,208],"identify":[8,209],"students'":[9,130,183],"faults.":[10,36,131],"However,":[11],"tests":[12,33,61],"may":[13],"not":[14,58,90],"detect":[15,35],"some":[16,198],"faults,":[17,95],"limiting":[18],"feedback,":[19],"and":[20,96,164,210],"providing":[21,65],"inaccurate":[22],"grades.":[23],"This":[24],"issue":[25],"can":[26,34,70,81,205],"be":[27,71,91],"mitigated":[28],"by":[29,60,93,145,157],"first":[30],"ensuring":[31],"that":[32,42,56,110,126,174,188,203],"Mutation":[37],"analysis":[38],"is":[39],"a":[40,48,149],"technique":[41],"generates":[43],"artificial":[44],"faulty":[45,180],"variants":[46],"of":[47,141,182],"program":[49],"for":[50,103,115,129],"this":[51,127,133],"purpose,":[52],"called":[53],"mutants.":[54],"Mutants":[55],"are":[57,112,139],"detected":[59],"reveal":[62],"their":[63,104],"inadequacies,":[64],"knowledge":[66],"on":[67,152],"how":[68],"they":[69],"improved.":[72],"By":[73],"using":[74],"mutants":[75,111,138,165,176,189,207],"improve":[77],"suites,":[79],"tutors":[80,204],"gain":[82],"the":[83,168,178],"confidence":[84],"that:":[85],"a)":[86],"generated":[87,166,175],"grades":[88],"will":[89,99],"biased":[92],"unidentified":[94],"b)":[97],"students":[98,158],"receive":[100],"appropriate":[101],"feedback":[102],"mistakes.":[105],"Existing":[106],"work":[107,123],"has":[108,124],"shown":[109,125],"suitable":[113],"substitutes":[114],"faults":[116],"in":[117,197,213],"real":[118],"world":[119],"software,":[120],"but":[121],"no":[122],"holds":[128],"In":[132],"paper,":[134],"we":[135],"investigate":[136],"whether":[137],"capable":[140],"replicating":[142],"mistakes":[143],"made":[144],"students.":[146],"We":[147,172,185],"conducted":[148],"quantitative":[150],"study":[151],"197":[153],"Java":[154],"classes":[155],"written":[156],"across":[159],"three":[160],"introductory":[161],"programming":[162],"assignments,":[163],"from":[167],"assignments'":[169],"model":[170],"solutions.":[171,184],"found":[173,187],"capture":[177],"observed":[179],"behaviour":[181],"also":[186],"better":[190],"assess":[191],"adequacy":[193],"than":[194],"code":[195],"coverage":[196],"cases.":[199],"Our":[200],"results":[201],"indicate":[202],"use":[206],"remedy":[211],"deficiencies":[212],"suites.":[216]},"counts_by_year":[{"year":2025,"cited_by_count":3},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":2}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}