{"id":"https://openalex.org/W4390621416","doi":"https://doi.org/10.1109/fie58773.2023.10342898","title":"Generating Multiple Choice Questions for Computing Courses Using Large Language Models","display_name":"Generating Multiple Choice Questions for Computing Courses Using Large Language Models","publication_year":2023,"publication_date":"2023-10-18","ids":{"openalex":"https://openalex.org/W4390621416","doi":"https://doi.org/10.1109/fie58773.2023.10342898"},"language":"en","primary_location":{"id":"doi:10.1109/fie58773.2023.10342898","is_oa":false,"landing_page_url":"https://doi.org/10.1109/fie58773.2023.10342898","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE Frontiers in Education Conference (FIE)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5074358617","display_name":"Andrew Tran","orcid":"https://orcid.org/0000-0002-0094-1113"},"institutions":[{"id":"https://openalex.org/I84392919","display_name":"Temple University","ror":"https://ror.org/00kx1jb78","country_code":"US","type":"education","lineage":["https://openalex.org/I84392919"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Andrew Tran","raw_affiliation_strings":["Temple University,Computer and Information Sciences,Philadelphia,USA","Computer and Information Sciences, Temple University, Philadelphia, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Temple University,Computer and Information Sciences,Philadelphia,USA","institution_ids":["https://openalex.org/I84392919"]},{"raw_affiliation_string":"Computer and Information Sciences, Temple University, Philadelphia, USA","institution_ids":["https://openalex.org/I84392919"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092862245","display_name":"Kenneth Angelikas","orcid":"https://orcid.org/0009-0001-1597-3513"},"institutions":[{"id":"https://openalex.org/I84392919","display_name":"Temple University","ror":"https://ror.org/00kx1jb78","country_code":"US","type":"education","lineage":["https://openalex.org/I84392919"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kenneth Angelikas","raw_affiliation_strings":["Temple University,Computer and Information Sciences,Philadelphia,USA","Computer and Information Sciences, Temple University, Philadelphia, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Temple University,Computer and Information Sciences,Philadelphia,USA","institution_ids":["https://openalex.org/I84392919"]},{"raw_affiliation_string":"Computer and Information Sciences, Temple University, Philadelphia, USA","institution_ids":["https://openalex.org/I84392919"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092862244","display_name":"Egi Rama","orcid":"https://orcid.org/0009-0005-9269-1455"},"institutions":[{"id":"https://openalex.org/I84392919","display_name":"Temple University","ror":"https://ror.org/00kx1jb78","country_code":"US","type":"education","lineage":["https://openalex.org/I84392919"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Egi Rama","raw_affiliation_strings":["Temple University,Computer and Information Sciences,Philadelphia,USA","Computer and Information Sciences, Temple University, Philadelphia, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Temple University,Computer and Information Sciences,Philadelphia,USA","institution_ids":["https://openalex.org/I84392919"]},{"raw_affiliation_string":"Computer and Information Sciences, Temple University, Philadelphia, USA","institution_ids":["https://openalex.org/I84392919"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113089841","display_name":"Chiku Okechukwu","orcid":null},"institutions":[{"id":"https://openalex.org/I84392919","display_name":"Temple University","ror":"https://ror.org/00kx1jb78","country_code":"US","type":"education","lineage":["https://openalex.org/I84392919"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chiku Okechukwu","raw_affiliation_strings":["Temple University,Computer and Information Sciences,Philadelphia,USA","Computer and Information Sciences, Temple University, Philadelphia, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Temple University,Computer and Information Sciences,Philadelphia,USA","institution_ids":["https://openalex.org/I84392919"]},{"raw_affiliation_string":"Computer and Information Sciences, Temple University, Philadelphia, USA","institution_ids":["https://openalex.org/I84392919"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014942043","display_name":"David H. Smith","orcid":"https://orcid.org/0000-0002-6572-4347"},"institutions":[{"id":"https://openalex.org/I84392919","display_name":"Temple University","ror":"https://ror.org/00kx1jb78","country_code":"US","type":"education","lineage":["https://openalex.org/I84392919"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"David H. Smith","raw_affiliation_strings":["Temple University,Computer and Information Sciences,Philadelphia,USA","Computer and Information Sciences, Temple University, Philadelphia, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Temple University,Computer and Information Sciences,Philadelphia,USA","institution_ids":["https://openalex.org/I84392919"]},{"raw_affiliation_string":"Computer and Information Sciences, Temple University, Philadelphia, USA","institution_ids":["https://openalex.org/I84392919"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5042822346","display_name":"Stephen MacNeil","orcid":"https://orcid.org/0000-0003-2781-6619"},"institutions":[{"id":"https://openalex.org/I84392919","display_name":"Temple University","ror":"https://ror.org/00kx1jb78","country_code":"US","type":"education","lineage":["https://openalex.org/I84392919"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Stephen MacNeil","raw_affiliation_strings":["Temple University,Computer and Information Sciences,Philadelphia,USA","Computer and Information Sciences, Temple University, Philadelphia, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Temple University,Computer and Information Sciences,Philadelphia,USA","institution_ids":["https://openalex.org/I84392919"]},{"raw_affiliation_string":"Computer and Information Sciences, Temple University, Philadelphia, USA","institution_ids":["https://openalex.org/I84392919"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I84392919"],"apc_list":null,"apc_paid":null,"fwci":16.1074,"has_fulltext":false,"cited_by_count":49,"citation_normalized_percentile":{"value":0.99395829,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"8"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.9926999807357788,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.9926999807357788,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12884","display_name":"Educational Assessment and Pedagogy","score":0.9914000034332275,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.9825999736785889,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7155075073242188},{"id":"https://openalex.org/keywords/multiple-choice","display_name":"Multiple choice","score":0.600594699382782},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.5358653664588928},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5202623605728149},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.5160991549491882},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5119756460189819},{"id":"https://openalex.org/keywords/craft","display_name":"Craft","score":0.4335312843322754},{"id":"https://openalex.org/keywords/class","display_name":"Class (philosophy)","score":0.415712833404541},{"id":"https://openalex.org/keywords/mathematics-education","display_name":"Mathematics education","score":0.39032310247421265},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.29085755348205566},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.2496817409992218},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.21000251173973083},{"id":"https://openalex.org/keywords/medicine","display_name":"Medicine","score":0.1665745973587036},{"id":"https://openalex.org/keywords/internal-medicine","display_name":"Internal medicine","score":0.09436401724815369}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7155075073242188},{"id":"https://openalex.org/C176730311","wikidata":"https://www.wikidata.org/wiki/Q867668","display_name":"Multiple choice","level":3,"score":0.600594699382782},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5358653664588928},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5202623605728149},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5160991549491882},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5119756460189819},{"id":"https://openalex.org/C2779732396","wikidata":"https://www.wikidata.org/wiki/Q2207288","display_name":"Craft","level":2,"score":0.4335312843322754},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.415712833404541},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.39032310247421265},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.29085755348205566},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.2496817409992218},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.21000251173973083},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.1665745973587036},{"id":"https://openalex.org/C126322002","wikidata":"https://www.wikidata.org/wiki/Q11180","display_name":"Internal medicine","level":1,"score":0.09436401724815369},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C3018023364","wikidata":"https://www.wikidata.org/wiki/Q425265","display_name":"Significant difference","level":2,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/fie58773.2023.10342898","is_oa":false,"landing_page_url":"https://doi.org/10.1109/fie58773.2023.10342898","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE Frontiers in Education Conference (FIE)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.4099999964237213,"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":51,"referenced_works":["https://openalex.org/W8102462","https://openalex.org/W144337011","https://openalex.org/W257906296","https://openalex.org/W611095317","https://openalex.org/W1975180387","https://openalex.org/W1981133620","https://openalex.org/W2002063893","https://openalex.org/W2020653198","https://openalex.org/W2065762074","https://openalex.org/W2074891565","https://openalex.org/W2077630385","https://openalex.org/W2098790754","https://openalex.org/W2105901442","https://openalex.org/W2106005988","https://openalex.org/W2151466713","https://openalex.org/W2158534319","https://openalex.org/W2249280543","https://openalex.org/W2760364374","https://openalex.org/W2876372830","https://openalex.org/W2997471824","https://openalex.org/W3005342969","https://openalex.org/W3032903422","https://openalex.org/W3144069799","https://openalex.org/W4206951823","https://openalex.org/W4211263275","https://openalex.org/W4225080353","https://openalex.org/W4283705032","https://openalex.org/W4289731614","https://openalex.org/W4291476001","https://openalex.org/W4292779060","https://openalex.org/W4303648559","https://openalex.org/W4306808680","https://openalex.org/W4311426581","https://openalex.org/W4319662928","https://openalex.org/W4323033785","https://openalex.org/W4323033814","https://openalex.org/W4323033853","https://openalex.org/W4323037544","https://openalex.org/W4323526518","https://openalex.org/W4366808937","https://openalex.org/W4382653968","https://openalex.org/W4382654294","https://openalex.org/W4386707723","https://openalex.org/W4390621457","https://openalex.org/W6622762108","https://openalex.org/W6675103397","https://openalex.org/W6778883912","https://openalex.org/W6791446462","https://openalex.org/W6845576796","https://openalex.org/W6855901311","https://openalex.org/W6959786010"],"related_works":["https://openalex.org/W2912168137","https://openalex.org/W2920882006","https://openalex.org/W2771752657","https://openalex.org/W2936836059","https://openalex.org/W1567987063","https://openalex.org/W2391860589","https://openalex.org/W2030429945","https://openalex.org/W114878902","https://openalex.org/W2061295771","https://openalex.org/W2115758876"],"abstract_inverted_index":{"Generating":[0],"high-quality":[1,104],"multiple-choice":[2],"questions":[3,24,201],"(MCQs)":[4],"is":[5,182],"a":[6,170,183,243],"time-consuming":[7],"activity":[8],"that":[9,69,172,230,256],"has":[10],"led":[11],"practitioners":[12],"and":[13,20,85,120,130,144,188,209],"researchers":[14],"to":[15,27,38,102,112,117,132,147,157,161,196,205,236,246,269,280],"develop":[16],"community":[17],"question":[18,168,226],"banks":[19],"reuse":[21],"the":[22,54,121,140,154,167,189,193,203,216,225],"same":[23,55],"from":[25,139],"semester":[26],"semester.":[28],"This":[29,228],"results":[30],"in":[31,58,64,249,263],"generic":[32],"MCQs":[33,45,116,135,138,156,221],"which":[34],"are":[35,50,70],"not":[36,175],"relevant":[37],"every":[39],"course.":[40],"Template-based":[41],"methods":[42],"for":[43,77,218,288],"generating":[44,81],"require":[46],"less":[47],"effort":[48],"but":[49],"similarly":[51],"limited.":[52],"At":[53],"time,":[56],"advances":[57],"natural":[59],"language":[60,66],"processing":[61],"have":[62],"resulted":[63],"large":[65],"models":[67,235],"(LLMs)":[68],"capable":[71],"of":[72,97,192,220],"doing":[73],"tasks":[74],"previously":[75,174],"reserved":[76],"people,":[78],"such":[79,240],"as":[80,241],"code,":[82],"code":[83],"explanations,":[84],"programming":[86],"assignments.":[87],"In":[88],"this":[89],"paper,":[90],"we":[91,178,275,285],"investigate":[92,179],"whether":[93,180,198],"these":[94,234],"generative":[95],"capabilities":[96],"LLMs":[98,278],"can":[99],"be":[100],"used":[101,126],"craft":[103],"M":[105],"CQs":[106],"more":[107,207],"efficiently,":[108],"thereby":[109],"enabling":[110],"instructors":[111,231],"focus":[113],"on":[114,137,166,224,260],"personalizing":[115],"each":[118],"course":[119],"associated":[122,194],"learning":[123],"goals.":[124],"We":[125,152,251,271],"two":[127],"LLMs,":[128],"GPT-3":[129,257],"GPT-4,":[131],"generate":[133,162],"isomorphic":[134],"based":[136,165,222],"Canterbury":[141],"Question":[142],"Bank":[143],"an":[145],"Introductory":[146],"Low-level":[148],"C":[149],"Programming":[150],"Course.":[151],"evaluated":[153],"resulting":[155],"assess":[158],"their":[159],"ability":[160],"correct":[163,267,282],"answers":[164,268],"stem,":[169],"task":[171],"was":[173],"possible.":[176],"Finally,":[177,284],"there":[181],"correlation":[184],"between":[185],"model":[186,204],"performance":[187],"discrimination":[190,200],"score":[191],"MCQ":[195],"understand":[197],"low":[199],"required":[202],"do":[206],"inference":[208],"therefore":[210],"perform":[211],"poorly.":[212],"GPT-4":[213],"correctly":[214],"generated":[215],"answer":[217],"78.5%":[219],"only":[223],"stem.":[227],"suggests":[229],"could":[232],"use":[233],"quickly":[237],"draft":[238],"quizzes,":[239],"during":[242],"live":[244],"class,":[245],"identify":[247],"misconceptions":[248],"real-time.":[250],"also":[252,272],"replicate":[253],"previous":[254],"findings":[255],"performs":[258],"poorly":[259],"answering,":[261],"or":[262],"our":[264],"case":[265],"generating,":[266],"MCQs.":[270],"present":[273],"cases":[274],"observed":[276],"where":[277],"struggled":[279],"produce":[281],"answers.":[283],"discuss":[286],"implications":[287],"computing":[289],"education.":[290]},"counts_by_year":[{"year":2026,"cited_by_count":8},{"year":2025,"cited_by_count":25},{"year":2024,"cited_by_count":15},{"year":2023,"cited_by_count":1}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}