{"id":"https://openalex.org/W2917018885","doi":"https://doi.org/10.1145/3287324.3287398","title":"On the Effect of Question Ordering on Performance and Confidence in Computer Science Examinations","display_name":"On the Effect of Question Ordering on Performance and Confidence in Computer Science Examinations","publication_year":2019,"publication_date":"2019-02-22","ids":{"openalex":"https://openalex.org/W2917018885","doi":"https://doi.org/10.1145/3287324.3287398","mag":"2917018885"},"language":"en","primary_location":{"id":"doi:10.1145/3287324.3287398","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3287324.3287398","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 50th ACM Technical Symposium on Computer Science Education","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5018644467","display_name":"Brian Harrington","orcid":"https://orcid.org/0000-0002-0734-9630"},"institutions":[{"id":"https://openalex.org/I185261750","display_name":"University of Toronto","ror":"https://ror.org/03dbr7087","country_code":"CA","type":"education","lineage":["https://openalex.org/I185261750"]},{"id":"https://openalex.org/I2799376586","display_name":"The Scarborough Hospital","ror":"https://ror.org/04pzgb662","country_code":"CA","type":"healthcare","lineage":["https://openalex.org/I2799376586"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Brian Harrington","raw_affiliation_strings":["University of Toronto Scarborough, Toronto, ON, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Toronto Scarborough, Toronto, ON, Canada","institution_ids":["https://openalex.org/I185261750","https://openalex.org/I2799376586"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029706768","display_name":"Jingyiran Li","orcid":null},"institutions":[{"id":"https://openalex.org/I185261750","display_name":"University of Toronto","ror":"https://ror.org/03dbr7087","country_code":"CA","type":"education","lineage":["https://openalex.org/I185261750"]},{"id":"https://openalex.org/I2799376586","display_name":"The Scarborough Hospital","ror":"https://ror.org/04pzgb662","country_code":"CA","type":"healthcare","lineage":["https://openalex.org/I2799376586"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Jingyiran Li","raw_affiliation_strings":["University of Toronto Scarborough, Toronto, ON, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Toronto Scarborough, Toronto, ON, Canada","institution_ids":["https://openalex.org/I185261750","https://openalex.org/I2799376586"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102955649","display_name":"Mohamed Moustafa","orcid":"https://orcid.org/0000-0002-6251-7115"},"institutions":[{"id":"https://openalex.org/I185261750","display_name":"University of Toronto","ror":"https://ror.org/03dbr7087","country_code":"CA","type":"education","lineage":["https://openalex.org/I185261750"]},{"id":"https://openalex.org/I2799376586","display_name":"The Scarborough Hospital","ror":"https://ror.org/04pzgb662","country_code":"CA","type":"healthcare","lineage":["https://openalex.org/I2799376586"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Mohamed Moustafa","raw_affiliation_strings":["University of Toronto Scarborough, Toronto, ON, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Toronto Scarborough, Toronto, ON, Canada","institution_ids":["https://openalex.org/I185261750","https://openalex.org/I2799376586"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076561791","display_name":"Marzieh Ahmadzadeh","orcid":"https://orcid.org/0000-0002-5653-6117"},"institutions":[{"id":"https://openalex.org/I185261750","display_name":"University of Toronto","ror":"https://ror.org/03dbr7087","country_code":"CA","type":"education","lineage":["https://openalex.org/I185261750"]},{"id":"https://openalex.org/I2799376586","display_name":"The Scarborough Hospital","ror":"https://ror.org/04pzgb662","country_code":"CA","type":"healthcare","lineage":["https://openalex.org/I2799376586"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Marzieh Ahmadzadeh","raw_affiliation_strings":["University of Toronto Scarborough, Toronto, ON, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Toronto Scarborough, Toronto, ON, Canada","institution_ids":["https://openalex.org/I185261750","https://openalex.org/I2799376586"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5028520471","display_name":"Nick Cheng","orcid":null},"institutions":[{"id":"https://openalex.org/I185261750","display_name":"University of Toronto","ror":"https://ror.org/03dbr7087","country_code":"CA","type":"education","lineage":["https://openalex.org/I185261750"]},{"id":"https://openalex.org/I2799376586","display_name":"The Scarborough Hospital","ror":"https://ror.org/04pzgb662","country_code":"CA","type":"healthcare","lineage":["https://openalex.org/I2799376586"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Nick Cheng","raw_affiliation_strings":["University of Toronto Scarborough, Toronto, ON, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Toronto Scarborough, Toronto, ON, Canada","institution_ids":["https://openalex.org/I185261750","https://openalex.org/I2799376586"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":4.3354,"has_fulltext":false,"cited_by_count":6,"citation_normalized_percentile":{"value":0.92987582,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"620","last_page":"626"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12515","display_name":"Gender and Technology in Education","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/3318","display_name":"Gender Studies"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T12515","display_name":"Gender and Technology in Education","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/3318","display_name":"Gender Studies"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.9972000122070312,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13034","display_name":"Information Systems Education and Curriculum Development","score":0.9842000007629395,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/perception","display_name":"Perception","score":0.6237478256225586},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5724242329597473},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5551674365997314},{"id":"https://openalex.org/keywords/structuring","display_name":"Structuring","score":0.5305607318878174},{"id":"https://openalex.org/keywords/class","display_name":"Class (philosophy)","score":0.4907711446285248},{"id":"https://openalex.org/keywords/sample","display_name":"Sample (material)","score":0.45589354634284973},{"id":"https://openalex.org/keywords/ask-price","display_name":"Ask price","score":0.4396348297595978},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.42919009923934937},{"id":"https://openalex.org/keywords/mathematics-education","display_name":"Mathematics education","score":0.42250511050224304},{"id":"https://openalex.org/keywords/cognitive-psychology","display_name":"Cognitive psychology","score":0.3564552068710327},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.21223360300064087},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.10143008828163147}],"concepts":[{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.6237478256225586},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5724242329597473},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5551674365997314},{"id":"https://openalex.org/C2775945657","wikidata":"https://www.wikidata.org/wiki/Q381442","display_name":"Structuring","level":2,"score":0.5305607318878174},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.4907711446285248},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.45589354634284973},{"id":"https://openalex.org/C90329073","wikidata":"https://www.wikidata.org/wiki/Q914232","display_name":"Ask price","level":2,"score":0.4396348297595978},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.42919009923934937},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.42250511050224304},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.3564552068710327},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.21223360300064087},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.10143008828163147},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.0},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0},{"id":"https://openalex.org/C136264566","wikidata":"https://www.wikidata.org/wiki/Q159810","display_name":"Economy","level":1,"score":0.0},{"id":"https://openalex.org/C10138342","wikidata":"https://www.wikidata.org/wiki/Q43015","display_name":"Finance","level":1,"score":0.0},{"id":"https://openalex.org/C169760540","wikidata":"https://www.wikidata.org/wiki/Q207011","display_name":"Neuroscience","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3287324.3287398","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3287324.3287398","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 50th ACM Technical Symposium on Computer Science Education","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.6600000262260437,"display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":17,"referenced_works":["https://openalex.org/W104632813","https://openalex.org/W1968811443","https://openalex.org/W2026379494","https://openalex.org/W2030002116","https://openalex.org/W2065040130","https://openalex.org/W2072390610","https://openalex.org/W2074613632","https://openalex.org/W2075977792","https://openalex.org/W2101929716","https://openalex.org/W2111210326","https://openalex.org/W2152135749","https://openalex.org/W2160836226","https://openalex.org/W2161855908","https://openalex.org/W2587238231","https://openalex.org/W2808863842","https://openalex.org/W2904588591","https://openalex.org/W3124551585"],"related_works":["https://openalex.org/W4251394462","https://openalex.org/W1580673008","https://openalex.org/W1510936208","https://openalex.org/W2486167009","https://openalex.org/W2168627904","https://openalex.org/W1595575899","https://openalex.org/W2515552481","https://openalex.org/W156769215","https://openalex.org/W2082963792","https://openalex.org/W2354233396"],"abstract_inverted_index":{"Most":[0],"computer":[1],"science":[2],"examinations":[3],"tend":[4],"to":[5,32,54,95,113,132,140,155],"start":[6],"with":[7],"the":[8,14,23,28,51,60,71,75,97,108,165,177,188,196,209,212,217,226],"easiest":[9],"questions":[10,135,213],"and":[11,179,228],"progress":[12],"towards":[13],"more":[15,100],"difficult":[16,101],"material.":[17],"Whether":[18],"this":[19,124],"is":[20,40,44,67,79],"because":[21],"of":[22,27,48,59,64,74,117,211,219,230],"highly":[24],"scaffolded":[25],"nature":[26],"course,":[29],"an":[30,88],"attempt":[31],"'ease":[33],"students":[34,94,139],"in',":[35],"or":[36,62,149],"simply":[37],"by":[38,70,83,136],"convention,":[39],"unclear.":[41],"However,":[42,195],"there":[43],"a":[45,65,169],"great":[46],"deal":[47],"data":[49],"from":[50],"psychology":[52],"literature":[53],"suggest":[55],"that":[56,82,164],"human":[57],"perception":[58,116],"difficulty":[61],"discomfort":[63],"task":[66,76],"disproportionately":[68],"affected":[69],"last":[72],"part":[73],"completed.":[77],"Therefore,":[78],"it":[80,103],"possible":[81],"structuring":[84],"our":[85],"exams":[86,142],"in":[87],"easy-to-hard":[89,145],"fashion,":[90],"we":[91],"are":[92],"causing":[93],"perceive":[96],"test":[98],"as":[99,147,191],"than":[102],"actually":[104],"is?":[105],"Could":[106],"changing":[107],"question":[109,166],"order":[110,210],"allow":[111],"us":[112],"change":[114],"students'":[115],"their":[118,157],"own":[119],"achievement?":[120],"What":[121],"effect":[122,175,182,197],"could":[123],"have":[125,215],"on":[126,159,176,183],"actual":[127],"performance?":[128],"This":[129],"paper":[130],"attempts":[131],"answer":[133],"these":[134],"randomly":[137],"assigning":[138],"write":[141],"ordered":[143],"either":[144],"(referred":[146],"'Easy-Difficult')":[148],"hard-to-easy":[150],"('Difficult-Easy),":[151],"then":[152],"ask":[153],"them":[154],"predict":[156],"marks":[158,185],"per-question":[160],"basis.":[161],"We":[162],"find":[163],"ordering":[167],"has":[168],"small":[170],"but":[171,222],"not":[172],"statistically":[173],"significant":[174,199],"performance,":[178],"virtually":[180],"no":[181],"predicted":[184],"when":[186],"treating":[187],"entire":[189],"class":[190],"one":[192],"unstratified":[193],"sample.":[194],"was":[198],"for":[200],"certain":[201],"subgroups":[202],"created":[203],"via":[204],"stratification.":[205],"In":[206],"particular,":[207],"swapping":[208],"may":[214],"hurt":[216],"performance":[218,227],"international":[220],"students,":[221],"significantly":[223],"raised":[224],"both":[225],"confidence":[229],"female":[231],"students.":[232]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":1},{"year":2021,"cited_by_count":1},{"year":2019,"cited_by_count":1}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}