{"id":"https://openalex.org/W4386718985","doi":"https://doi.org/10.1007/s40593-023-00372-z","title":"Can ChatGPT Pass High School Exams on English Language Comprehension?","display_name":"Can ChatGPT Pass High School Exams on English Language Comprehension?","publication_year":2023,"publication_date":"2023-09-13","ids":{"openalex":"https://openalex.org/W4386718985","doi":"https://doi.org/10.1007/s40593-023-00372-z"},"language":"en","primary_location":{"id":"doi:10.1007/s40593-023-00372-z","doi":"https://doi.org/10.1007/s40593-023-00372-z","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s40593-023-00372-z","pdf_url":"https://link.springer.com/content/pdf/10.1007/s40593-023-00372-z.pdf","source":{"id":"https://openalex.org/S171267539","display_name":"International Journal of Artificial Intelligence in Education","issn_l":"1560-4292","issn":["1560-4292","1560-4306"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","dongbi-b","erih-plus","jufo-1","norway-1","poland-100","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Journal of Artificial Intelligence in Education","raw_type":"journal-article"},"volume":null,"type":"article","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://link.springer.com/content/pdf/10.1007/s40593-023-00372-z.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5037003206","display_name":"Joost de Winter","orcid":"https://orcid.org/0000-0002-1281-8200","observed_orcids":["https://orcid.org/0000-0002-1281-8200"]},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":true,"raw_author_name":"Joost C.F. de Winter","raw_affiliation_strings":["Cognitive Robotics Department, Delft University of Technology, Delft, Netherlands"],"raw_orcid":"https://orcid.org/0000-0002-1281-8200","affiliations":[{"raw_affiliation_string":"Cognitive Robotics Department, Delft University of Technology, Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5037003206"],"corresponding_institution_ids":["https://openalex.org/I98358874"],"apc_list":{"value":2790,"currency":"USD","value_usd":2790},"apc_paid":{"value":2790,"currency":"USD","value_usd":2790},"fwci":42.6544,"has_fulltext":true,"cited_by_count":147,"citation_normalized_percentile":{"value":0.9985133,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":"34","issue":"3","first_page":"915","last_page":"930"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T14414","display_name":"Artificial Intelligence in Education","score":0.9193999767303467,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T14414","display_name":"Artificial Intelligence in Education","score":0.9193999767303467,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.019099999219179153,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T11902","display_name":"Intelligent Tutoring Systems and Adaptive Learning","score":0.014000000432133675,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/chatgpt","display_name":"ChatGPT","score":0.9990000128746033},{"id":"https://openalex.org/keywords/gpt-4","display_name":"GPT-4","score":0.9150000214576721},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.9810000061988831},{"id":"https://openalex.org/keywords/english-reading-comprehension","display_name":"English reading comprehension","score":0.7329999804496765},{"id":"https://openalex.org/keywords/educational-assessment","display_name":"educational assessment","score":0.5870000123977661},{"id":"https://openalex.org/keywords/artificial-intelligence-in-education","display_name":"artificial intelligence in education","score":0.4729999899864197},{"id":"https://openalex.org/keywords/temperature-parameter","display_name":"temperature parameter","score":0.6330000162124634},{"id":"https://openalex.org/keywords/bootstrap-method","display_name":"bootstrapping","score":0.6480000019073486}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5599995851516724},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.4702790081501007},{"id":"https://openalex.org/C2778780117","wikidata":"https://www.wikidata.org/wiki/Q3269423","display_name":"Reading comprehension","level":3,"score":0.4587031602859497},{"id":"https://openalex.org/C511192102","wikidata":"https://www.wikidata.org/wiki/Q5156948","display_name":"Comprehension","level":2,"score":0.44174566864967346},{"id":"https://openalex.org/C554936623","wikidata":"https://www.wikidata.org/wiki/Q199657","display_name":"Reading (process)","level":2,"score":0.4248199760913849},{"id":"https://openalex.org/C2987496018","wikidata":"https://www.wikidata.org/wiki/Q1860","display_name":"English language","level":2,"score":0.4176400303840637},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.34084033966064453},{"id":"https://openalex.org/C509550671","wikidata":"https://www.wikidata.org/wiki/Q126945","display_name":"Medical education","level":1,"score":0.3317207098007202},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.21435123682022095},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.15632501244544983},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.13221335411071777},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1007/s40593-023-00372-z","doi":"https://doi.org/10.1007/s40593-023-00372-z","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s40593-023-00372-z","pdf_url":"https://link.springer.com/content/pdf/10.1007/s40593-023-00372-z.pdf","source":{"id":"https://openalex.org/S171267539","display_name":"International Journal of Artificial Intelligence in Education","issn_l":"1560-4292","issn":["1560-4292","1560-4306"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","dongbi-b","erih-plus","jufo-1","norway-1","poland-100","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Journal of Artificial Intelligence in Education","raw_type":"journal-article"},{"id":"pmh:oai:tudelft.nl:uuid:9f37ed35-47e3-4c4b-83af-165c6d1d4a91","doi":null,"is_oa":true,"landing_page_url":"http://resolver.tudelft.nl/uuid:9f37ed35-47e3-4c4b-83af-165c6d1d4a91","pdf_url":null,"source":{"id":"https://openalex.org/S4306400906","display_name":"Research Repository (Delft University of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I98358874","host_organization_name":"Delft University of Technology","host_organization_lineage":["https://openalex.org/I98358874"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"journal article"}],"best_oa_location":{"id":"doi:10.1007/s40593-023-00372-z","doi":"https://doi.org/10.1007/s40593-023-00372-z","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s40593-023-00372-z","pdf_url":"https://link.springer.com/content/pdf/10.1007/s40593-023-00372-z.pdf","source":{"id":"https://openalex.org/S171267539","display_name":"International Journal of Artificial Intelligence in Education","issn_l":"1560-4292","issn":["1560-4292","1560-4306"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","dongbi-b","erih-plus","jufo-1","norway-1","poland-100","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Journal of Artificial Intelligence in Education","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Quality education","id":"https://metadata.un.org/sdg/4","score":0.975600004196167}],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4386718985.pdf","grobid_xml":"https://content.openalex.org/works/W4386718985.grobid-xml"},"referenced_works_count":39,"referenced_works":["https://openalex.org/W3132494213","https://openalex.org/W4221161695","https://openalex.org/W4283026156","https://openalex.org/W4292200252","https://openalex.org/W4311481166","https://openalex.org/W4312051216","https://openalex.org/W4312212221","https://openalex.org/W4313384047","https://openalex.org/W4313422136","https://openalex.org/W4313564799","https://openalex.org/W4317910584","https://openalex.org/W4319460874","https://openalex.org/W4319662928","https://openalex.org/W4321524280","https://openalex.org/W4323035510","https://openalex.org/W4323043848","https://openalex.org/W4323655724","https://openalex.org/W4324046518","https://openalex.org/W4327811249","https://openalex.org/W4327811957","https://openalex.org/W4360836968","https://openalex.org/W4362700315","https://openalex.org/W4362723020","https://openalex.org/W4366565380","https://openalex.org/W4377855098","https://openalex.org/W4381587445","https://openalex.org/W4385302156","https://openalex.org/W4386724748","https://openalex.org/W4387823622","https://openalex.org/W4388735460","https://openalex.org/W4389519108","https://openalex.org/W4390940237","https://openalex.org/W4392681182","https://openalex.org/W4392773006","https://openalex.org/W4401042689","https://openalex.org/W4403863303","https://openalex.org/W6847547502","https://openalex.org/W6939554937","https://openalex.org/W7133223925"],"related_works":["https://openalex.org/W4381082115","https://openalex.org/W4400514768","https://openalex.org/W4380994371","https://openalex.org/W7165114437","https://openalex.org/W4401586423","https://openalex.org/W4400972825","https://openalex.org/W7165191964","https://openalex.org/W4390940237","https://openalex.org/W7222093620","https://openalex.org/W4402819599"],"abstract_inverted_index":{"Launched":[0],"in":[1,34,89,163,185],"late":[2,45],"November":[3],"2022,":[4,47],"ChatGPT,":[5,125],"a":[6,65,121,140,145,151,169],"large":[7,206],"language":[8,207],"model":[9,175],"chatbot,":[10],"has":[11],"garnered":[12],"considerable":[13],"attention.":[14],"However,":[15,93],"ongoing":[16],"questions":[17,52],"remain":[18],"regarding":[19],"its":[20],"capabilities.":[21],"In":[22,44,112],"this":[23],"study,":[24],"ChatGPT":[25,55,63,94,203],"was":[26,110,117,127],"used":[27],"to":[28,60,76,78,98,134],"complete":[29],"national":[30],"high":[31],"school":[32],"exams":[33,133],"the":[35,38,50,54,71,79,87,90,107,131,135,173,186,200],"Netherlands":[36],"on":[37,70,209],"topic":[39],"of":[40,68,74,82,124,142,178,202],"English":[41],"reading":[42],"comprehension.":[43],"December":[46],"we":[48],"submitted":[49,130],"exam":[51,88],"through":[53,157],"web":[56],"interface":[57],"(GPT-3.5).":[58],"According":[59],"official":[61],"norms,":[62],"achieved":[64,139],"mean":[66,80],"grade":[67,81,109],"7.3":[69],"Dutch":[72],"scale":[73],"1":[75],"10\u2014comparable":[77],"all":[83],"students":[84],"who":[85],"took":[86],"Netherlands,":[91],"6.99.":[92],"occasionally":[95],"required":[96],"re-prompting":[97],"arrive":[99],"at":[100],"an":[101],"explicit":[102],"answer;":[103],"without":[104,144],"these":[105],"nudges,":[106],"overall":[108,187],"6.5.":[111],"March":[113],"2023,":[114],"API":[115],"access":[116],"made":[118],"available,":[119],"and":[120,137,204],"new":[122],"version":[123],"GPT-4,":[126],"released.":[128],"We":[129],"same":[132],"API,":[136],"GPT-4":[138,174],"score":[141],"8.3":[143],"need":[146],"for":[147],"re-prompting.":[148],"Additionally,":[149],"employing":[150],"bootstrapping":[152],"method":[153],"that":[154],"incorporated":[155],"randomness":[156],"ChatGPT\u2019s":[158],"\u2018temperature\u2019":[159],"parameter":[160],"proved":[161],"effective":[162],"self-identifying":[164],"potentially":[165],"incorrect":[166],"answers.":[167],"Finally,":[168],"re-assessment":[170],"conducted":[171],"with":[172],"updated":[176],"as":[177],"June":[179],"2023":[180],"showed":[181],"no":[182],"substantial":[183],"change":[184],"score.":[188],"The":[189],"present":[190],"findings":[191],"highlight":[192],"significant":[193],"opportunities":[194],"but":[195],"also":[196],"raise":[197],"concerns":[198],"about":[199],"impact":[201],"similar":[205],"models":[208],"educational":[210],"assessment.":[211]},"counts_by_year":[{"year":2026,"cited_by_count":24},{"year":2025,"cited_by_count":35},{"year":2024,"cited_by_count":50},{"year":2023,"cited_by_count":38}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2023-09-14T00:00:00"}