{"id":"https://openalex.org/W4385570371","doi":"https://doi.org/10.18653/v1/2023.findings-acl.29","title":"A Systematic Study and Comprehensive Evaluation of ChatGPT on Benchmark Datasets","display_name":"A Systematic Study and Comprehensive Evaluation of ChatGPT on Benchmark Datasets","publication_year":2023,"publication_date":"2023-01-01","ids":{"openalex":"https://openalex.org/W4385570371","doi":"https://doi.org/10.18653/v1/2023.findings-acl.29"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2023.findings-acl.29","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.findings-acl.29","pdf_url":"https://aclanthology.org/2023.findings-acl.29.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2023","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2023.findings-acl.29.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5069546450","display_name":"Md Tahmid Rahman Laskar","orcid":"https://orcid.org/0009-0001-7880-299X"},"institutions":[{"id":"https://openalex.org/I192455969","display_name":"York University","ror":"https://ror.org/05fq50484","country_code":"CA","type":"education","lineage":["https://openalex.org/I192455969"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Md Tahmid Rahman Laskar","raw_affiliation_strings":["York University,","Dialpad Canada Inc.,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"York University,","institution_ids":["https://openalex.org/I192455969"]},{"raw_affiliation_string":"Dialpad Canada Inc.,","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034756201","display_name":"M Saiful Bari","orcid":null},"institutions":[{"id":"https://openalex.org/I172675005","display_name":"Nanyang Technological University","ror":"https://ror.org/02e7b5302","country_code":"SG","type":"education","lineage":["https://openalex.org/I172675005"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"M Saiful Bari","raw_affiliation_strings":["Nanyang Technological University,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Nanyang Technological University,","institution_ids":["https://openalex.org/I172675005"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047113835","display_name":"Mizanur Rahman","orcid":"https://orcid.org/0000-0003-1128-753X"},"institutions":[{"id":"https://openalex.org/I125133608","display_name":"Royal Bank of Canada","ror":"https://ror.org/03hgnwx26","country_code":"CA","type":"other","lineage":["https://openalex.org/I125133608"]},{"id":"https://openalex.org/I192455969","display_name":"York University","ror":"https://ror.org/05fq50484","country_code":"CA","type":"education","lineage":["https://openalex.org/I192455969"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Mizanur Rahman","raw_affiliation_strings":["Royal Bank of Canada, $ Salesforce Research","York University,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Royal Bank of Canada, $ Salesforce Research","institution_ids":["https://openalex.org/I125133608"]},{"raw_affiliation_string":"York University,","institution_ids":["https://openalex.org/I192455969"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043888616","display_name":"Md Amran Hossen Bhuiyan","orcid":null},"institutions":[{"id":"https://openalex.org/I172675005","display_name":"Nanyang Technological University","ror":"https://ror.org/02e7b5302","country_code":"SG","type":"education","lineage":["https://openalex.org/I172675005"]},{"id":"https://openalex.org/I192455969","display_name":"York University","ror":"https://ror.org/05fq50484","country_code":"CA","type":"education","lineage":["https://openalex.org/I192455969"]}],"countries":["CA","SG"],"is_corresponding":false,"raw_author_name":"Md Amran Hossen Bhuiyan","raw_affiliation_strings":["Nanyang Technological University,","York University,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Nanyang Technological University,","institution_ids":["https://openalex.org/I172675005"]},{"raw_affiliation_string":"York University,","institution_ids":["https://openalex.org/I192455969"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005443526","display_name":"Shafiq Joty","orcid":"https://orcid.org/0000-0002-9222-2641"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shafiq Joty","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5000409439","display_name":"Jimmy Xiangji Huang","orcid":"https://orcid.org/0000-0003-1292-1491"},"institutions":[{"id":"https://openalex.org/I192455969","display_name":"York University","ror":"https://ror.org/05fq50484","country_code":"CA","type":"education","lineage":["https://openalex.org/I192455969"]}],"countries":["CA"],"is_corresponding":true,"raw_author_name":"Jimmy Huang","raw_affiliation_strings":["York University,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"York University,","institution_ids":["https://openalex.org/I192455969"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5000409439"],"corresponding_institution_ids":["https://openalex.org/I192455969"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":78,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"431","last_page":"469"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9961000084877014,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9961000084877014,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.9807999730110168,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.9437999725341797,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/automatic-summarization","display_name":"Automatic summarization","score":0.8316572904586792},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.816382884979248},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8095539808273315},{"id":"https://openalex.org/keywords/variety","display_name":"Variety (cybernetics)","score":0.5525967478752136},{"id":"https://openalex.org/keywords/strengths-and-weaknesses","display_name":"Strengths and weaknesses","score":0.5382551550865173},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5369479656219482},{"id":"https://openalex.org/keywords/machine-translation","display_name":"Machine translation","score":0.5065826177597046},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5021471977233887},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.4813835918903351},{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.47655990719795227},{"id":"https://openalex.org/keywords/generative-grammar","display_name":"Generative grammar","score":0.4534863829612732},{"id":"https://openalex.org/keywords/ground-truth","display_name":"Ground truth","score":0.41733527183532715},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.35655778646469116}],"concepts":[{"id":"https://openalex.org/C170858558","wikidata":"https://www.wikidata.org/wiki/Q1394144","display_name":"Automatic summarization","level":2,"score":0.8316572904586792},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.816382884979248},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8095539808273315},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.5525967478752136},{"id":"https://openalex.org/C63882131","wikidata":"https://www.wikidata.org/wiki/Q17122954","display_name":"Strengths and weaknesses","level":2,"score":0.5382551550865173},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5369479656219482},{"id":"https://openalex.org/C203005215","wikidata":"https://www.wikidata.org/wiki/Q79798","display_name":"Machine translation","level":2,"score":0.5065826177597046},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5021471977233887},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4813835918903351},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.47655990719795227},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.4534863829612732},{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.41733527183532715},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.35655778646469116},{"id":"https://openalex.org/C162853370","wikidata":"https://www.wikidata.org/wiki/Q39809","display_name":"Marketing","level":1,"score":0.0},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2023.findings-acl.29","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.findings-acl.29","pdf_url":"https://aclanthology.org/2023.findings-acl.29.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2023","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2023.findings-acl.29","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.findings-acl.29","pdf_url":"https://aclanthology.org/2023.findings-acl.29.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2023","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.44999998807907104}],"awards":[],"funders":[{"id":"https://openalex.org/F4320334593","display_name":"Natural Sciences and Engineering Research Council of Canada","ror":"https://ror.org/01h531d29"}],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4385570371.pdf"},"referenced_works_count":154,"referenced_works":["https://openalex.org/W1599016936","https://openalex.org/W2113459411","https://openalex.org/W2145755360","https://openalex.org/W2154652894","https://openalex.org/W2252136820","https://openalex.org/W2257408573","https://openalex.org/W2512924740","https://openalex.org/W2606964149","https://openalex.org/W2606974598","https://openalex.org/W2804897457","https://openalex.org/W2888482885","https://openalex.org/W2890894339","https://openalex.org/W2898662126","https://openalex.org/W2903193068","https://openalex.org/W2911489562","https://openalex.org/W2912924812","https://openalex.org/W2923014074","https://openalex.org/W2925863688","https://openalex.org/W2927103915","https://openalex.org/W2943552823","https://openalex.org/W2946609015","https://openalex.org/W2948902769","https://openalex.org/W2949615363","https://openalex.org/W2950819771","https://openalex.org/W2952087486","https://openalex.org/W2963323070","https://openalex.org/W2963339397","https://openalex.org/W2963457723","https://openalex.org/W2963466651","https://openalex.org/W2963526187","https://openalex.org/W2963527228","https://openalex.org/W2963903950","https://openalex.org/W2965373594","https://openalex.org/W2970062726","https://openalex.org/W2970771982","https://openalex.org/W2971258845","https://openalex.org/W2978017171","https://openalex.org/W2988937804","https://openalex.org/W2989743967","https://openalex.org/W2996428491","https://openalex.org/W2996908057","https://openalex.org/W2997090102","https://openalex.org/W2998617917","https://openalex.org/W3014635508","https://openalex.org/W3029927342","https://openalex.org/W3034850762","https://openalex.org/W3034999214","https://openalex.org/W3035579820","https://openalex.org/W3046375318","https://openalex.org/W3047185145","https://openalex.org/W3083410900","https://openalex.org/W3104748221","https://openalex.org/W3118109963","https://openalex.org/W3118423943","https://openalex.org/W3118781290","https://openalex.org/W3121904249","https://openalex.org/W3126675481","https://openalex.org/W3134642945","https://openalex.org/W3153675281","https://openalex.org/W3168656614","https://openalex.org/W3168867926","https://openalex.org/W3171639395","https://openalex.org/W3174770825","https://openalex.org/W3196731672","https://openalex.org/W3205068155","https://openalex.org/W3213074177","https://openalex.org/W4205991051","https://openalex.org/W4220732108","https://openalex.org/W4225591000","https://openalex.org/W4225619898","https://openalex.org/W4226146865","https://openalex.org/W4226278401","https://openalex.org/W4226399820","https://openalex.org/W4253067820","https://openalex.org/W4280652569","https://openalex.org/W4281690148","https://openalex.org/W4283026156","https://openalex.org/W4283768109","https://openalex.org/W4284713203","https://openalex.org/W4285178342","https://openalex.org/W4285222512","https://openalex.org/W4285225959","https://openalex.org/W4285292339","https://openalex.org/W4286987939","https://openalex.org/W4287024925","https://openalex.org/W4287854446","https://openalex.org/W4287887705","https://openalex.org/W4287888693","https://openalex.org/W4287900772","https://openalex.org/W4288089799","https://openalex.org/W4294790344","https://openalex.org/W4297253404","https://openalex.org/W4297435087","https://openalex.org/W4298051537","https://openalex.org/W4303443398","https://openalex.org/W4303648904","https://openalex.org/W4307123345","https://openalex.org/W4308163975","https://openalex.org/W4311642023","https://openalex.org/W4312051216","https://openalex.org/W4313447114","https://openalex.org/W4313559133","https://openalex.org/W4317547570","https://openalex.org/W4318899036","https://openalex.org/W4319793302","https://openalex.org/W4319985185","https://openalex.org/W4319991848","https://openalex.org/W4320167623","https://openalex.org/W4320854779","https://openalex.org/W4321472057","https://openalex.org/W4321472314","https://openalex.org/W4321524280","https://openalex.org/W4322708560","https://openalex.org/W4322718191","https://openalex.org/W4323980273","https://openalex.org/W4327811957","https://openalex.org/W4360584537","https://openalex.org/W4361019453","https://openalex.org/W4361019538","https://openalex.org/W4361806442","https://openalex.org/W4362511131","https://openalex.org/W4362679631","https://openalex.org/W4363671637","https://openalex.org/W4364385323","https://openalex.org/W4366459745","https://openalex.org/W4366559955","https://openalex.org/W4366850809","https://openalex.org/W4366999665","https://openalex.org/W4366999773","https://openalex.org/W4367365799","https://openalex.org/W4367860052","https://openalex.org/W4368250038","https://openalex.org/W4375949262","https://openalex.org/W4376122680","https://openalex.org/W4376167348","https://openalex.org/W4378805624","https://openalex.org/W4379374334","https://openalex.org/W4385245566","https://openalex.org/W4385565111","https://openalex.org/W4385571124","https://openalex.org/W4385571140","https://openalex.org/W4385571157","https://openalex.org/W4385572482","https://openalex.org/W4385734111","https://openalex.org/W4386298264","https://openalex.org/W4389315156","https://openalex.org/W4389518968","https://openalex.org/W4389519239","https://openalex.org/W4389519325","https://openalex.org/W4389519396","https://openalex.org/W4389519817","https://openalex.org/W4389520124","https://openalex.org/W4389523957","https://openalex.org/W4389524534"],"related_works":["https://openalex.org/W2366403280","https://openalex.org/W1495108544","https://openalex.org/W2091301346","https://openalex.org/W3148229873","https://openalex.org/W2150160875","https://openalex.org/W4242223894","https://openalex.org/W1517524280","https://openalex.org/W4284703357","https://openalex.org/W3046127589","https://openalex.org/W4287704324"],"abstract_inverted_index":{"The":[0],"development":[1],"of":[2,14,31,54,103,119,163,168,198,214],"large":[3],"language":[4],"models":[5],"(LLMs)":[6],"such":[7],"as":[8],"ChatGPT":[9,83,104,120,148,160],"has":[10],"brought":[11],"a":[12,51,135,165,195,211],"lot":[13],"attention":[15],"recently.":[16],"However,":[17],"their":[18],"evaluation":[19,53,102,155],"in":[20,93,105,121,147,175,217],"the":[21,29,33,41,100,115,185,208],"benchmark":[22,177],"academic":[23,59],"datasets":[24],"remains":[25],"under-explored":[26],"due":[27],"to":[28,49,113,139,187],"difficulty":[30],"evaluating":[32],"generative":[34],"outputs":[35],"produced":[36],"by":[37],"this":[38,45,205],"model":[39],"against":[40],"ground":[42],"truth.":[43],"In":[44,108],"paper,":[46],"we":[47,81,144],"aim":[48],"present":[50],"thorough":[52,196],"ChatGPT's":[55,199],"performance":[56,174,200],"on":[57],"diverse":[58,202],"datasets,":[60,178],"covering":[61],"tasks":[62,86,123],"like":[63],"question-answering,":[64],"text":[65],"summarization,":[66],"code":[67],"generation,":[68],"commonsense":[69],"reasoning,":[70],"mathematical":[71],"problem-solving,":[72],"machine":[73],"translation,":[74],"bias":[75],"detection,":[76],"and":[77,87,117,124,149,170],"ethical":[78],"considerations.":[79],"Specifically,":[80],"evaluate":[82],"across":[84,201],"140":[85],"analyze":[88],"255K":[89],"responses":[90],"it":[91,179],"generates":[92],"these":[94],"datasets.":[95],"This":[96],"makes":[97],"our":[98,110],"work":[99],"largest":[101],"NLP":[106,203],"benchmarks.":[107],"short,":[109],"study":[111],"aims":[112],"validate":[114],"strengths":[116],"weaknesses":[118],"various":[122],"provide":[125],"insights":[126],"for":[127,210],"future":[128],"research":[129],"using":[130],"LLMs.":[131],"We":[132],"also":[133],"report":[134],"new":[136],"emergent":[137],"ability":[138,186],"follow":[140],"multi-query":[141],"instructions":[142],"that":[143,157],"mostly":[145],"found":[146],"other":[150],"instruction-tuned":[151],"models.":[152],"Our":[153],"extensive":[154],"shows":[156],"even":[158],"though":[159],"is":[161,180],"capable":[162],"performing":[164],"wide":[166],"variety":[167],"tasks,":[169,204],"may":[171],"obtain":[172],"impressive":[173],"several":[176],"still":[181],"far":[182],"from":[183],"achieving":[184],"reliably":[188],"solve":[189],"many":[190],"challenging":[191],"tasks.":[192],"By":[193],"providing":[194],"assessment":[197],"paper":[206],"sets":[207],"stage":[209],"targeted":[212],"deployment":[213],"ChatGPT-like":[215],"LLMs":[216],"real-world":[218],"applications.":[219]},"counts_by_year":[{"year":2026,"cited_by_count":8},{"year":2025,"cited_by_count":19},{"year":2024,"cited_by_count":30},{"year":2023,"cited_by_count":21}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}