{"id":"https://openalex.org/W4405434047","doi":"https://doi.org/10.1145/3706468.3706530","title":"Does Multiple Choice Have a Future in the Age of Generative AI? A Posttest-only RCT","display_name":"Does Multiple Choice Have a Future in the Age of Generative AI? A Posttest-only RCT","publication_year":2025,"publication_date":"2025-02-21","ids":{"openalex":"https://openalex.org/W4405434047","doi":"https://doi.org/10.1145/3706468.3706530"},"language":"en","primary_location":{"id":"doi:10.1145/3706468.3706530","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3706468.3706530","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 15th International Learning Analytics and Knowledge Conference","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3706468.3706530","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5039927056","display_name":"Danielle R. Thomas","orcid":"https://orcid.org/0000-0001-8196-3252"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Danielle R Thomas","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA","SANJIT KAKARLA, Carnegie Mellon University, USA","Boyuan","ERIN GATZ, Carnegie Mellon University, USA","JIONGHAO LIN, Carnegie Mellon University, USA","Carnegie Mellon University, USA","Carnegie Mellon University, Pittsburgh, PA, USA;"],"raw_orcid":"https://orcid.org/0000-0001-8196-3252","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"SANJIT KAKARLA, Carnegie Mellon University, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Boyuan","institution_ids":[]},{"raw_affiliation_string":"ERIN GATZ, Carnegie Mellon University, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"JIONGHAO LIN, Carnegie Mellon University, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Carnegie Mellon University, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA;","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037442366","display_name":"Conrad Borchers","orcid":"https://orcid.org/0000-0003-3437-8979"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Conrad Borchers","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA","SANJIT KAKARLA, Carnegie Mellon University, USA","Boyuan","Carnegie Mellon University, Pittsburgh, PA, USA;","ERIN GATZ, Carnegie Mellon University, USA","Carnegie Mellon University, USA","JIONGHAO LIN, Carnegie Mellon University, USA"],"raw_orcid":"https://orcid.org/0000-0003-3437-8979","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"SANJIT KAKARLA, Carnegie Mellon University, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Boyuan","institution_ids":[]},{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA;","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"ERIN GATZ, Carnegie Mellon University, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Carnegie Mellon University, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"JIONGHAO LIN, Carnegie Mellon University, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093701234","display_name":"Sanjit Kakarla","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sanjit Kakarla","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA"],"raw_orcid":"https://orcid.org/0009-0007-6508-8647","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021833341","display_name":"Jionghao Lin","orcid":"https://orcid.org/0000-0003-3320-3907"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jionghao Lin","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA"],"raw_orcid":"https://orcid.org/0000-0003-3320-3907","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054927819","display_name":"Shambhavi Bhushan","orcid":"https://orcid.org/0009-0004-3695-2334"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shambhavi Bhushan","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA","SANJIT KAKARLA, Carnegie Mellon University, USA","Carnegie Mellon University, USA","ERIN GATZ, Carnegie Mellon University, USA","Carnegie Mellon University, Pittsburgh, PA, USA;","Boyuan","JIONGHAO LIN, Carnegie Mellon University, USA"],"raw_orcid":"https://orcid.org/0009-0004-3695-2334","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"SANJIT KAKARLA, Carnegie Mellon University, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Carnegie Mellon University, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"ERIN GATZ, Carnegie Mellon University, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA;","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Boyuan","institution_ids":[]},{"raw_affiliation_string":"JIONGHAO LIN, Carnegie Mellon University, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005075201","display_name":"Boyuan Guo","orcid":"https://orcid.org/0000-0002-8303-4649"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Boyuan Guo","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA","JIONGHAO LIN, Carnegie Mellon University, USA","ERIN GATZ, Carnegie Mellon University, USA","Boyuan","Carnegie Mellon University, USA","SANJIT KAKARLA, Carnegie Mellon University, USA","Carnegie Mellon University, Pittsburgh, PA, USA;"],"raw_orcid":"https://orcid.org/0000-0002-8303-4649","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"JIONGHAO LIN, Carnegie Mellon University, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"ERIN GATZ, Carnegie Mellon University, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Boyuan","institution_ids":[]},{"raw_affiliation_string":"Carnegie Mellon University, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"SANJIT KAKARLA, Carnegie Mellon University, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA;","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092458766","display_name":"Erin Gatz","orcid":"https://orcid.org/0000-0002-6880-5740"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Erin Gatz","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA","SANJIT KAKARLA, Carnegie Mellon University, USA","Carnegie Mellon University, Pittsburgh, PA, USA;","JIONGHAO LIN, Carnegie Mellon University, USA","Carnegie Mellon University, USA","Boyuan","ERIN GATZ, Carnegie Mellon University, USA"],"raw_orcid":"https://orcid.org/0000-0002-6880-5740","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"SANJIT KAKARLA, Carnegie Mellon University, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA;","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"JIONGHAO LIN, Carnegie Mellon University, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Carnegie Mellon University, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Boyuan","institution_ids":[]},{"raw_affiliation_string":"ERIN GATZ, Carnegie Mellon University, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5062550465","display_name":"Kenneth R. Koedinger","orcid":"https://orcid.org/0000-0002-5850-4768"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kenneth R Koedinger","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA","Boyuan","JIONGHAO LIN, Carnegie Mellon University, USA","Carnegie Mellon University, Pittsburgh, PA, USA;","SANJIT KAKARLA, Carnegie Mellon University, USA","Carnegie Mellon University, USA","ERIN GATZ, Carnegie Mellon University, USA"],"raw_orcid":"https://orcid.org/0000-0002-5850-4768","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Boyuan","institution_ids":[]},{"raw_affiliation_string":"JIONGHAO LIN, Carnegie Mellon University, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA;","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"SANJIT KAKARLA, Carnegie Mellon University, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Carnegie Mellon University, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"ERIN GATZ, Carnegie Mellon University, USA","institution_ids":["https://openalex.org/I74973139"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5039927056"],"corresponding_institution_ids":["https://openalex.org/I74973139"],"apc_list":null,"apc_paid":null,"fwci":9.1879,"has_fulltext":false,"cited_by_count":5,"citation_normalized_percentile":{"value":0.97164295,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":95,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"494","last_page":"504"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11902","display_name":"Intelligent Tutoring Systems and Adaptive Learning","score":0.9943000078201294,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11902","display_name":"Intelligent Tutoring Systems and Adaptive Learning","score":0.9943000078201294,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.9757999777793884,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12002","display_name":"Computability, Logic, AI Algorithms","score":0.9696000218391418,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/randomized-controlled-trial","display_name":"Randomized controlled trial","score":0.762059211730957},{"id":"https://openalex.org/keywords/generative-grammar","display_name":"Generative grammar","score":0.637764573097229},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.4913560450077057},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.32961505651474},{"id":"https://openalex.org/keywords/medicine","display_name":"Medicine","score":0.24027758836746216},{"id":"https://openalex.org/keywords/internal-medicine","display_name":"Internal medicine","score":0.06471481919288635}],"concepts":[{"id":"https://openalex.org/C168563851","wikidata":"https://www.wikidata.org/wiki/Q1436668","display_name":"Randomized controlled trial","level":2,"score":0.762059211730957},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.637764573097229},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4913560450077057},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.32961505651474},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.24027758836746216},{"id":"https://openalex.org/C126322002","wikidata":"https://www.wikidata.org/wiki/Q11180","display_name":"Internal medicine","level":1,"score":0.06471481919288635}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3706468.3706530","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3706468.3706530","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 15th International Learning Analytics and Knowledge Conference","raw_type":"proceedings-article"},{"id":"pmh:oai:hub.hku.hk:10722/358757","is_oa":false,"landing_page_url":"https://hub.hku.hk/handle/10722/358757","pdf_url":null,"source":{"id":"https://openalex.org/S4377196271","display_name":"The HKU Scholars Hub (University of Hong Kong)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I889458895","host_organization_name":"University of Hong Kong","host_organization_lineage":["https://openalex.org/I889458895"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":null,"raw_type":"Conference_Paper"},{"id":"pmh:oai:arXiv.org:2412.10267","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2412.10267","pdf_url":"https://arxiv.org/pdf/2412.10267","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.1145/3706468.3706530","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3706468.3706530","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 15th International Learning Analytics and Knowledge Conference","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":25,"referenced_works":["https://openalex.org/W242171001","https://openalex.org/W1982877624","https://openalex.org/W2038558034","https://openalex.org/W2108617939","https://openalex.org/W2113386642","https://openalex.org/W2887398393","https://openalex.org/W2995822004","https://openalex.org/W3041179138","https://openalex.org/W3157580258","https://openalex.org/W3199008897","https://openalex.org/W4250727746","https://openalex.org/W4281662144","https://openalex.org/W4300280802","https://openalex.org/W4321442087","https://openalex.org/W4381956223","https://openalex.org/W4383180725","https://openalex.org/W4385570141","https://openalex.org/W4391578379","https://openalex.org/W4392202731","https://openalex.org/W4392484100","https://openalex.org/W4399639995","https://openalex.org/W4400210772","https://openalex.org/W4400642837","https://openalex.org/W4400643369","https://openalex.org/W4400668042"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2355769538","https://openalex.org/W2081816252","https://openalex.org/W2380075625","https://openalex.org/W2015216653","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433"],"abstract_inverted_index":{"The":[0],"role":[1],"of":[2,81,98,163,179],"multiple-choice":[3],"questions":[4,29],"(MCQs)":[5],"as":[6,129],"effective":[7],"learning":[8,104,139],"tools":[9],"has":[10],"been":[11],"debated":[12],"in":[13,25,37,57,112],"past":[14],"research.":[15],"While":[16],"MCQs":[17,48,127],"are":[18,30,63,128],"widely":[19],"used":[20,32],"due":[21],"to":[22,51,120,189],"their":[23],"ease":[24],"grading,":[26],"open":[27,92,135,151],"response":[28,93,136],"increasingly":[31],"for":[33,42,138,161,171],"instruction,":[34],"given":[35],"advances":[36],"large":[38],"language":[39],"models":[40,158],"(LLMs)":[41],"automated":[43],"grading.":[44],"This":[45,174],"study":[46,175],"evaluates":[47],"effectiveness":[49],"relative":[50],"open-response":[52],"questions,":[53],"both":[54],"individually":[55],"and":[56,95,131,155,186,192],"combination,":[58],"on":[59,69],"learning.":[60],"These":[61,123],"activities":[62],"embedded":[64],"within":[65],"six":[66],"tutor":[67],"lessons":[68],"advocacy.":[70],"Using":[71],"a":[72,96,177],"posttest-only":[73],"randomized":[74],"control":[75],"design,":[76],"we":[77,149],"compare":[78],"the":[79,113],"performance":[80],"234":[82],"tutors":[83,111],"(790":[84],"lesson":[85,180],"completions)":[86],"across":[87,106],"three":[88],"conditions:":[89],"MCQ":[90,114],"only,":[91,94],"combination":[97],"both.":[99],"We":[100],"find":[101],"no":[102],"significant":[103],"differences":[105],"conditions":[107],"at":[108],"posttest,":[109],"but":[110],"condition":[115],"took":[116],"significantly":[117],"less":[118],"time":[119,142],"complete":[121],"instruction.":[122],"findings":[124],"suggest":[125],"that":[126],"effective,":[130],"more":[132],"efficient,":[133],"than":[134],"tasks":[137],"when":[140],"practice":[141],"is":[143,169],"limited.":[144],"To":[145],"further":[146,167],"enhance":[147],"efficiency,":[148],"autograded":[150],"responses":[152],"using":[153],"GPT-4o":[154],"GPT-4-turbo.":[156],"GPT":[157],"demonstrate":[159],"proficiency":[160],"purposes":[162],"low-stakes":[164],"assessment,":[165],"though":[166],"research":[168],"needed":[170],"broader":[172],"use.":[173],"contributes":[176],"dataset":[178],"log":[181],"data,":[182],"human":[183],"annotation":[184],"rubrics,":[185],"LLM":[187],"prompts":[188],"promote":[190],"transparency":[191],"reproducibility.":[193]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":4}],"updated_date":"2026-06-02T09:04:35.204637","created_date":"2024-12-17T00:00:00"}