{"id":"https://openalex.org/W3213424100","doi":"https://doi.org/10.1145/3488042.3488045","title":"Wrong Answers for Wrong Reasons: The\u00a0Risks\u00a0of\u00a0Ad\u00a0Hoc\u00a0Instruments","display_name":"Wrong Answers for Wrong Reasons: The\u00a0Risks\u00a0of\u00a0Ad\u00a0Hoc\u00a0Instruments","publication_year":2021,"publication_date":"2021-11-10","ids":{"openalex":"https://openalex.org/W3213424100","doi":"https://doi.org/10.1145/3488042.3488045","mag":"3213424100"},"language":"en","primary_location":{"id":"doi:10.1145/3488042.3488045","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3488042.3488045","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 21st Koli Calling International Conference on Computing Education Research","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5026393158","display_name":"Luca Chiodini","orcid":"https://orcid.org/0000-0002-2712-9248"},"institutions":[{"id":"https://openalex.org/I57201433","display_name":"Universit\u00e0 della Svizzera italiana","ror":"https://ror.org/03c4atk17","country_code":"CH","type":"education","lineage":["https://openalex.org/I57201433"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Luca Chiodini","raw_affiliation_strings":["Software Institute - Universit\u00e0 della Svizzera italiana, Switzerland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Software Institute - Universit\u00e0 della Svizzera italiana, Switzerland","institution_ids":["https://openalex.org/I57201433"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5078770676","display_name":"Matthias Hauswirth","orcid":"https://orcid.org/0000-0001-5527-5931"},"institutions":[{"id":"https://openalex.org/I57201433","display_name":"Universit\u00e0 della Svizzera italiana","ror":"https://ror.org/03c4atk17","country_code":"CH","type":"education","lineage":["https://openalex.org/I57201433"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Matthias Hauswirth","raw_affiliation_strings":["Software Institute - Universit\u00e0 della Svizzera italiana, Switzerland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Software Institute - Universit\u00e0 della Svizzera italiana, Switzerland","institution_ids":["https://openalex.org/I57201433"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I57201433"],"apc_list":null,"apc_paid":null,"fwci":0.586,"has_fulltext":false,"cited_by_count":6,"citation_normalized_percentile":{"value":0.69645498,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"11"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11902","display_name":"Intelligent Tutoring Systems and Adaptive Learning","score":0.996999979019165,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11902","display_name":"Intelligent Tutoring Systems and Adaptive Learning","score":0.996999979019165,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.9966999888420105,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.9919999837875366,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/correctness","display_name":"Correctness","score":0.7085447311401367},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6908251643180847},{"id":"https://openalex.org/keywords/soundness","display_name":"Soundness","score":0.6464985609054565},{"id":"https://openalex.org/keywords/post-hoc","display_name":"Post hoc","score":0.5781025886535645},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.5400022268295288},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.5311656594276428},{"id":"https://openalex.org/keywords/post-hoc-analysis","display_name":"Post-hoc analysis","score":0.5086129903793335},{"id":"https://openalex.org/keywords/medicine","display_name":"Medicine","score":0.10177743434906006}],"concepts":[{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.7085447311401367},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6908251643180847},{"id":"https://openalex.org/C39920170","wikidata":"https://www.wikidata.org/wiki/Q693083","display_name":"Soundness","level":2,"score":0.6464985609054565},{"id":"https://openalex.org/C2992886853","wikidata":"https://www.wikidata.org/wiki/Q18381816","display_name":"Post hoc","level":2,"score":0.5781025886535645},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.5400022268295288},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.5311656594276428},{"id":"https://openalex.org/C67761136","wikidata":"https://www.wikidata.org/wiki/Q2105849","display_name":"Post-hoc analysis","level":2,"score":0.5086129903793335},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.10177743434906006},{"id":"https://openalex.org/C120665830","wikidata":"https://www.wikidata.org/wiki/Q14620","display_name":"Optics","level":1,"score":0.0},{"id":"https://openalex.org/C199343813","wikidata":"https://www.wikidata.org/wiki/Q12128","display_name":"Dentistry","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C126322002","wikidata":"https://www.wikidata.org/wiki/Q11180","display_name":"Internal medicine","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3488042.3488045","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3488042.3488045","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 21st Koli Calling International Conference on Computing Education Research","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.8600000143051147,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":54,"referenced_works":["https://openalex.org/W1511583024","https://openalex.org/W1971621932","https://openalex.org/W2065325564","https://openalex.org/W2074344724","https://openalex.org/W2087517248","https://openalex.org/W2118662626","https://openalex.org/W2144644874","https://openalex.org/W2161498332","https://openalex.org/W2279714893","https://openalex.org/W2284289670","https://openalex.org/W2460079845","https://openalex.org/W2460612746","https://openalex.org/W2512013293","https://openalex.org/W2513097092","https://openalex.org/W2513356680","https://openalex.org/W2729185598","https://openalex.org/W2729530250","https://openalex.org/W2730342518","https://openalex.org/W2734210175","https://openalex.org/W2745758802","https://openalex.org/W2746128040","https://openalex.org/W2748071319","https://openalex.org/W2748486817","https://openalex.org/W2766230222","https://openalex.org/W2809339505","https://openalex.org/W2809671282","https://openalex.org/W2886150761","https://openalex.org/W2886511030","https://openalex.org/W2900834637","https://openalex.org/W2900974709","https://openalex.org/W2910926550","https://openalex.org/W2915075304","https://openalex.org/W2963095537","https://openalex.org/W2963266459","https://openalex.org/W2963378383","https://openalex.org/W2964684033","https://openalex.org/W2964982419","https://openalex.org/W2965145667","https://openalex.org/W2966150097","https://openalex.org/W2966268134","https://openalex.org/W2996163751","https://openalex.org/W3005723079","https://openalex.org/W3033104398","https://openalex.org/W3033286267","https://openalex.org/W3033383133","https://openalex.org/W3033884854","https://openalex.org/W3033913109","https://openalex.org/W3047890686","https://openalex.org/W3109586032","https://openalex.org/W3110054500","https://openalex.org/W3110452114","https://openalex.org/W3177397114","https://openalex.org/W4230330965","https://openalex.org/W4298884035"],"related_works":["https://openalex.org/W2402392332","https://openalex.org/W2182025991","https://openalex.org/W3036293936","https://openalex.org/W2090568800","https://openalex.org/W2795815529","https://openalex.org/W2104816286","https://openalex.org/W2372155899","https://openalex.org/W2775903035","https://openalex.org/W2528834793","https://openalex.org/W3213424100"],"abstract_inverted_index":{"To":[0],"evaluate":[1],"novel":[2],"pedagogies,":[3],"approaches,":[4,173],"and":[5,69,139,165,175],"tools,":[6],"Computer":[7,102],"Science":[8,103],"Education":[9,104],"researchers":[10,37],"often":[11,113],"conduct":[12],"experiments":[13,29,94],"to":[14,56,73,137,145,150,178],"look":[15],"for":[16],"differences":[17],"among":[18],"groups":[19],"treated":[20],"with":[21,114,174],"different":[22],"interventions.":[23],"The":[24],"methodological":[25],"rigor":[26],"of":[27,33,49,65,92,123,143,158,183],"such":[28,50,124],"affects":[30],"the":[31,34,36,53,63,70,90,96,121,141,167,171,181,188],"soundness":[32],"conclusions":[35,186],"can":[38],"draw.":[39],"In":[40],"this":[41],"paper":[42],"we":[43,61],"focus":[44],"on":[45],"a":[46,79,134,156],"central":[47],"aspect":[48],"experimental":[51],"research:":[52],"instruments":[54,68,88],"used":[55,126],"assess":[57,140],"participants\u2019":[58],"knowledge.":[59],"Specifically,":[60],"study":[62],"use":[64],"ad":[66,110],"hoc":[67,111],"risks":[71,122],"due":[72],"their":[74],"insufficient":[75],"validation.":[76],"We":[77,119,132,161],"present":[78],"literature":[80],"survey":[81],"that":[82,180],"highlights":[83],"how,":[84],"even":[85],"though":[86],"standardized":[87],"exist,":[89],"majority":[91],"published":[93],"in":[95],"last":[97],"five":[98],"years":[99],"at":[100],"major":[101],"conferences":[105],"carries":[106],"out":[107],"pre/post-tests":[108],"using":[109,170],"instruments,":[112],"multiple-choice":[115,130,146],"as":[116,155],"question":[117],"type.":[118],"demonstrate":[120],"commonly":[125],"but":[127],"insufficiently":[128],"validated":[129],"instruments.":[131],"propose":[133],"richer":[135],"way":[136],"analyze":[138,166],"correctness":[142],"answers":[144,169],"questions,":[147],"requiring":[148],"participants":[149],"add":[151],"brief":[152],"explanation":[153],"texts":[154],"justification":[157],"each":[159],"answer.":[160],"run":[162],"an":[163],"experiment":[164],"collected":[168],"two":[172],"without":[176],"explanations,":[177],"show":[179],"risk":[182],"drawing":[184],"opposite":[185],"from":[187],"statistical":[189],"analysis":[190],"is":[191],"real.":[192]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":1}],"updated_date":"2026-07-30T17:31:21.811387","created_date":"2025-10-10T00:00:00"}