{"id":"https://openalex.org/W4410987602","doi":"https://doi.org/10.1109/educon62633.2025.11016485","title":"Can We Trust AI Chatbots to Teach University Physics? A Performance Comparison of AI Chatbots","display_name":"Can We Trust AI Chatbots to Teach University Physics? A Performance Comparison of AI Chatbots","publication_year":2025,"publication_date":"2025-04-22","ids":{"openalex":"https://openalex.org/W4410987602","doi":"https://doi.org/10.1109/educon62633.2025.11016485"},"language":"en","primary_location":{"id":"doi:10.1109/educon62633.2025.11016485","is_oa":false,"landing_page_url":"https://doi.org/10.1109/educon62633.2025.11016485","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE Global Engineering Education Conference (EDUCON)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5061912834","display_name":"V\u00edctor Robledo-Rella","orcid":"https://orcid.org/0000-0002-7892-0515"},"institutions":[{"id":"https://openalex.org/I98461037","display_name":"Tecnol\u00f3gico de Monterrey","ror":"https://ror.org/03ayjn504","country_code":"MX","type":"education","lineage":["https://openalex.org/I98461037"]}],"countries":["MX"],"is_corresponding":true,"raw_author_name":"V\u00edctor Robledo-Rella","raw_affiliation_strings":["School of Engineering and Science Tecnologico de Monterrey,Mexico, City"],"affiliations":[{"raw_affiliation_string":"School of Engineering and Science Tecnologico de Monterrey,Mexico, City","institution_ids":["https://openalex.org/I98461037"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089120357","display_name":"Andr\u00e9s Gonz\u00e1lez-Nucamendi","orcid":"https://orcid.org/0000-0001-5707-2767"},"institutions":[{"id":"https://openalex.org/I98461037","display_name":"Tecnol\u00f3gico de Monterrey","ror":"https://ror.org/03ayjn504","country_code":"MX","type":"education","lineage":["https://openalex.org/I98461037"]}],"countries":["MX"],"is_corresponding":false,"raw_author_name":"Andr\u00e9s Gonzalez-Nucamendi","raw_affiliation_strings":["School of Engineering and Science Tecnologico de Monterrey,Mexico, City"],"affiliations":[{"raw_affiliation_string":"School of Engineering and Science Tecnologico de Monterrey,Mexico, City","institution_ids":["https://openalex.org/I98461037"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047274269","display_name":"Luis Neri","orcid":"https://orcid.org/0000-0002-3331-8854"},"institutions":[{"id":"https://openalex.org/I98461037","display_name":"Tecnol\u00f3gico de Monterrey","ror":"https://ror.org/03ayjn504","country_code":"MX","type":"education","lineage":["https://openalex.org/I98461037"]}],"countries":["MX"],"is_corresponding":false,"raw_author_name":"Luis Neri","raw_affiliation_strings":["School of Engineering and Science Tecnologico de Monterrey,Mexico, City"],"affiliations":[{"raw_affiliation_string":"School of Engineering and Science Tecnologico de Monterrey,Mexico, City","institution_ids":["https://openalex.org/I98461037"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047263907","display_name":"Rosa Mar\u00eda Guadalupe Garc\u00eda-Castel\u00e1n","orcid":"https://orcid.org/0000-0002-8362-522X"},"institutions":[{"id":"https://openalex.org/I98461037","display_name":"Tecnol\u00f3gico de Monterrey","ror":"https://ror.org/03ayjn504","country_code":"MX","type":"education","lineage":["https://openalex.org/I98461037"]}],"countries":["MX"],"is_corresponding":false,"raw_author_name":"Rosa M.G. Garc\u00eda-Castel\u00e1n","raw_affiliation_strings":["School of Engineering and Science Tecnologico de Monterrey,Mexico, City"],"affiliations":[{"raw_affiliation_string":"School of Engineering and Science Tecnologico de Monterrey,Mexico, City","institution_ids":["https://openalex.org/I98461037"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018252798","display_name":"Julieta Noguez","orcid":"https://orcid.org/0000-0002-6000-3452"},"institutions":[{"id":"https://openalex.org/I98461037","display_name":"Tecnol\u00f3gico de Monterrey","ror":"https://ror.org/03ayjn504","country_code":"MX","type":"education","lineage":["https://openalex.org/I98461037"]}],"countries":["MX"],"is_corresponding":false,"raw_author_name":"Julieta Noguez","raw_affiliation_strings":["School of Engineering and Science Tecnologico de Monterrey,Mexico, City"],"affiliations":[{"raw_affiliation_string":"School of Engineering and Science Tecnologico de Monterrey,Mexico, City","institution_ids":["https://openalex.org/I98461037"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5042155265","display_name":"Jorge Valverde-Rebaza","orcid":"https://orcid.org/0000-0001-8664-9692"},"institutions":[{"id":"https://openalex.org/I98461037","display_name":"Tecnol\u00f3gico de Monterrey","ror":"https://ror.org/03ayjn504","country_code":"MX","type":"education","lineage":["https://openalex.org/I98461037"]}],"countries":["MX"],"is_corresponding":false,"raw_author_name":"Jorge Valverde-Rebaza","raw_affiliation_strings":["School of Engineering and Science Tecnologico de Monterrey,Mexico, City"],"affiliations":[{"raw_affiliation_string":"School of Engineering and Science Tecnologico de Monterrey,Mexico, City","institution_ids":["https://openalex.org/I98461037"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5061912834"],"corresponding_institution_ids":["https://openalex.org/I98461037"],"apc_list":null,"apc_paid":null,"fwci":1.8424,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.87946055,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":95},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"7"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.9839000105857849,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.9839000105857849,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12128","display_name":"AI in Service Interactions","score":0.9571999907493591,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.9251999855041504,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/chatbot","display_name":"Chatbot","score":0.7229611873626709},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.4878294765949249},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.33587056398391724},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.32962095737457275},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.32248765230178833}],"concepts":[{"id":"https://openalex.org/C2779041454","wikidata":"https://www.wikidata.org/wiki/Q870780","display_name":"Chatbot","level":2,"score":0.7229611873626709},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4878294765949249},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.33587056398391724},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.32962095737457275},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.32248765230178833}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/educon62633.2025.11016485","is_oa":false,"landing_page_url":"https://doi.org/10.1109/educon62633.2025.11016485","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE Global Engineering Education Conference (EDUCON)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":15,"referenced_works":["https://openalex.org/W4364377381","https://openalex.org/W4377234196","https://openalex.org/W4377855098","https://openalex.org/W4387493711","https://openalex.org/W4387812493","https://openalex.org/W4387936944","https://openalex.org/W4390082902","https://openalex.org/W4390981665","https://openalex.org/W4391575364","https://openalex.org/W4398243313","https://openalex.org/W4399614351","https://openalex.org/W4399776384","https://openalex.org/W4404950861","https://openalex.org/W4404972124","https://openalex.org/W6858543681"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W4383501580","https://openalex.org/W4214931137","https://openalex.org/W4313813117","https://openalex.org/W4382052417","https://openalex.org/W3192088754","https://openalex.org/W4387007686","https://openalex.org/W3084631705"],"abstract_inverted_index":{"Generative":[0],"artificial":[1],"intelligence":[2],"(AI)":[3],"is":[4,39,65],"transforming":[5],"the":[6,54,103,118,186,203,209,238],"learning":[7],"and":[8,11,29,61,95,106,121,219,244,270,285],"teaching":[9],"landscape,":[10],"large":[12],"language":[13],"models,":[14],"such":[15],"as":[16,276],"ChatGPT,":[17],"Gemini,":[18],"or":[19],"Copilot,":[20],"are":[21],"demonstrating":[22],"their":[23,35,47,69,280],"outstanding":[24],"ability":[25,70],"to":[26,43,71,97,109,117,171,195,226,258],"hold":[27],"conversations":[28],"generate":[30],"relevant":[31],"content.":[32],"However,":[33],"given":[34,102,200,231],"probabilistic":[36],"nature,":[37],"it":[38,136,245],"currently":[40],"not":[41,214],"possible":[42],"trust":[44],"completely":[45],"in":[46,262],"academic":[48],"performance.":[49,188],"A":[50,81,233],"detailed":[51],"comparison":[52],"of":[53,56,76,84,145,166,176,251,283],"accuracy":[55,123,144,191,222,250],"two":[57],"AI":[58,256],"chatbots,":[59],"ChatGPT-4o":[60],"Gemini":[62,155],"1.5":[63],"Pro":[64],"presented":[66],"by":[67,235],"measuring":[68],"correctly":[72],"answer":[73],"multiple-choice":[74,260],"questions":[75,86,261],"standard":[77],"undergraduate":[78,263],"physics":[79,264],"questionnaires.":[80],"total":[82],"sample":[83],"790":[85],"was":[87,101,107,125,137,178,241],"analyzed,":[88],"covering":[89],"13":[90],"mechanics'":[91],"themes,":[92],"from":[93,237],"units":[94],"measurements":[96],"gravitation.":[98],"Each":[99],"chatbot":[100],"same":[104,187],"prompt":[105],"asked":[108],"solve":[110,259],"each":[111,128],"question.":[112],"Their":[113],"answers":[114],"were":[115],"compared":[116],"correct":[119],"answers,":[120],"an":[122,142,248],"indicator":[124],"derived":[126],"for":[127,198,229],"chatbot.":[129],"In":[130],"partial":[131],"agreement":[132],"with":[133,208],"similar":[134],"studies,":[135],"found":[138],"that":[139],"ChatGPT":[140,177,206],"achieved":[141,156,247],"overall":[143,249],"only":[146,246],"58":[147],"(\u00b17)":[148],"points":[149,197,228],"(on":[150],"a":[151,161,199,216,230,277],"0\u2013100":[152],"scale),":[153],"while":[154],"79":[157],"(\u00b16),":[158],"which":[159],"represents":[160],"36%":[162],"improvement.":[163],"An":[164],"analysis":[165],"variances":[167],"proved":[168],"this":[169],"difference":[170],"be":[172],"significant.":[173],"The":[174],"stability":[175],"also":[179,242],"tested":[180],"through":[181],"different":[182],"runs,":[183],"yielding":[184],"basically":[185],"Its":[189],"average":[190,221],"may":[192,223],"vary":[193,224],"up":[194,225],"11":[196],"theme.":[201,232],"On":[202],"other":[204],"hand,":[205],"running":[207],"Wolfram":[210],"Alpha":[211],"plugin":[212],"did":[213],"achieve":[215],"systematic":[217],"improvement,":[218],"its":[220],"17":[227],"PhysicsGPT":[234],"pulsr.co.uk":[236],"OpenAI":[239],"gallery":[240],"tested,":[243],"57.":[252],"When":[253],"using":[254],"these":[255,274],"chatbots":[257],"topics":[265],"requiring":[266],"mathematical":[267],"procedures,":[268],"teachers":[269],"students":[271],"should":[272],"take":[273],"results":[275],"warning":[278],"about":[279],"current":[281],"lack":[282],"precision":[284],"potential":[286],"misleadingness.":[287]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}