{"id":"https://openalex.org/W4417483163","doi":"https://doi.org/10.1186/s40537-025-01330-3","title":"Meta-analysis of large language models: benchmarking DeepSeek-R1 against ChatGPT, Gemini, Qwen, and LLaMA","display_name":"Meta-analysis of large language models: benchmarking DeepSeek-R1 against ChatGPT, Gemini, Qwen, and LLaMA","publication_year":2025,"publication_date":"2025-12-19","ids":{"openalex":"https://openalex.org/W4417483163","doi":"https://doi.org/10.1186/s40537-025-01330-3"},"language":"en","primary_location":{"id":"doi:10.1186/s40537-025-01330-3","is_oa":true,"landing_page_url":"https://doi.org/10.1186/s40537-025-01330-3","pdf_url":null,"source":{"id":"https://openalex.org/S2737955091","display_name":"Journal Of Big Data","issn_l":"2196-1115","issn":["2196-1115"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Big Data","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1186/s40537-025-01330-3","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5113073456","display_name":"Shafique Ahmed Awan","orcid":"https://orcid.org/0009-0002-3901-2095"},"institutions":[{"id":"https://openalex.org/I12469534","display_name":"Quaid-i-Azam University","ror":"https://ror.org/04s9hft57","country_code":"PK","type":"education","lineage":["https://openalex.org/I12469534"]}],"countries":["PK"],"is_corresponding":true,"raw_author_name":"Shafique Ahmed Awan","raw_affiliation_strings":["Department of CS, Quaid-I-Azam University, Islamabad, Pakistan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of CS, Quaid-I-Azam University, Islamabad, Pakistan","institution_ids":["https://openalex.org/I12469534"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103250431","display_name":"Muazzam A. Khan","orcid":"https://orcid.org/0000-0002-2713-5605"},"institutions":[{"id":"https://openalex.org/I12469534","display_name":"Quaid-i-Azam University","ror":"https://ror.org/04s9hft57","country_code":"PK","type":"education","lineage":["https://openalex.org/I12469534"]}],"countries":["PK"],"is_corresponding":false,"raw_author_name":"Muazzam Ali Khan Khattak","raw_affiliation_strings":["Department of CS, Quaid-I-Azam University, Islamabad, Pakistan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of CS, Quaid-I-Azam University, Islamabad, Pakistan","institution_ids":["https://openalex.org/I12469534"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080933055","display_name":"Abdullah Ayub Khan","orcid":"https://orcid.org/0000-0003-2838-7641"},"institutions":[{"id":"https://openalex.org/I59225215","display_name":"Bahria University","ror":"https://ror.org/02v8d7770","country_code":"PK","type":"education","lineage":["https://openalex.org/I59225215"]}],"countries":["PK"],"is_corresponding":false,"raw_author_name":"Abdullah Ayub Khan","raw_affiliation_strings":["Department of Computer Science, Bahria University Karachi Campus, Karachi, 75260, Pakistan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, Bahria University Karachi Campus, Karachi, 75260, Pakistan","institution_ids":["https://openalex.org/I59225215"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025030253","display_name":"Anwar Ali Sathio","orcid":"https://orcid.org/0000-0001-5059-0166"},"institutions":[{"id":"https://openalex.org/I1308893265","display_name":"Shaheed Benazir Bhutto University","ror":"https://ror.org/02zwhz281","country_code":"PK","type":"education","lineage":["https://openalex.org/I1308893265"]}],"countries":["PK"],"is_corresponding":false,"raw_author_name":"Anwar Ali Sathio","raw_affiliation_strings":["Department of CS&IT, Benazir Bhutto Shaheed University, Karachi, Sindh, Pakistan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of CS&IT, Benazir Bhutto Shaheed University, Karachi, Sindh, Pakistan","institution_ids":["https://openalex.org/I1308893265"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087784805","display_name":"Jamil Abedalrahim Jamil Alsayaydeh","orcid":"https://orcid.org/0000-0002-9768-4925"},"institutions":[{"id":"https://openalex.org/I32589535","display_name":"Technical University of Malaysia Malacca","ror":"https://ror.org/01xb6rs26","country_code":"MY","type":"education","lineage":["https://openalex.org/I32589535"]}],"countries":["MY"],"is_corresponding":false,"raw_author_name":"Jamil Abedalrahim Jamil Alsayaydeh","raw_affiliation_strings":["Department of Engineering Technology, Fakulti Teknologi Dan Kejuruteraan Elektronik Dan Komputer (FTKEK), Universiti Teknikal Malaysia Melaka (UTeM), 76100, Melaka, Malaysia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Engineering Technology, Fakulti Teknologi Dan Kejuruteraan Elektronik Dan Komputer (FTKEK), Universiti Teknikal Malaysia Melaka (UTeM), 76100, Melaka, Malaysia","institution_ids":["https://openalex.org/I32589535"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5098658993","display_name":"Rex Bacarra","orcid":"https://orcid.org/0000-0001-9247-4374"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rex Bacarra","raw_affiliation_strings":["Department of General Education and Foundation, Rabdan Academy, Abu Dhabi, United Arab Emirates"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of General Education and Foundation, Rabdan Academy, Abu Dhabi, United Arab Emirates","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5077466854","display_name":"Safarudin Gazali Herawan","orcid":"https://orcid.org/0000-0002-5725-8075"},"institutions":[{"id":"https://openalex.org/I166073570","display_name":"Binus University","ror":"https://ror.org/03zmf4s77","country_code":"ID","type":"education","lineage":["https://openalex.org/I166073570"]}],"countries":["ID"],"is_corresponding":false,"raw_author_name":"Safarudin Gazali Herawan","raw_affiliation_strings":["Industrial Engineering Department, Faculty of Engineering, Bina Nusantara University, Jakarta, 11480, Indonesia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Industrial Engineering Department, Faculty of Engineering, Bina Nusantara University, Jakarta, 11480, Indonesia","institution_ids":["https://openalex.org/I166073570"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5120852572","display_name":"Rehman Aziz","orcid":null},"institutions":[{"id":"https://openalex.org/I12469534","display_name":"Quaid-i-Azam University","ror":"https://ror.org/04s9hft57","country_code":"PK","type":"education","lineage":["https://openalex.org/I12469534"]}],"countries":["PK"],"is_corresponding":false,"raw_author_name":"Rehman Aziz","raw_affiliation_strings":["Department of CS, Quaid-I-Azam University, Islamabad, Pakistan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of CS, Quaid-I-Azam University, Islamabad, Pakistan","institution_ids":["https://openalex.org/I12469534"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5113073456"],"corresponding_institution_ids":["https://openalex.org/I12469534"],"apc_list":{"value":1060,"currency":"GBP","value_usd":1300},"apc_paid":{"value":1060,"currency":"GBP","value_usd":1300},"fwci":0.5261,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.77701414,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":96,"max":98},"biblio":{"volume":"13","issue":"1","first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.550000011920929,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.550000011920929,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.09799999743700027,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.05990000069141388,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.8996999859809875},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.6570000052452087},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.5440999865531921},{"id":"https://openalex.org/keywords/coding","display_name":"Coding (social sciences)","score":0.4390999972820282},{"id":"https://openalex.org/keywords/efficient-energy-use","display_name":"Efficient energy use","score":0.37279999256134033},{"id":"https://openalex.org/keywords/turbo","display_name":"Turbo","score":0.3646000027656555}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.8996999859809875},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8299000263214111},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.6570000052452087},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5440999865531921},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.5065000057220459},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.4390999972820282},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4108000099658966},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4004000127315521},{"id":"https://openalex.org/C2742236","wikidata":"https://www.wikidata.org/wiki/Q924713","display_name":"Efficient energy use","level":2,"score":0.37279999256134033},{"id":"https://openalex.org/C2776240298","wikidata":"https://www.wikidata.org/wiki/Q1138513","display_name":"Turbo","level":2,"score":0.3646000027656555},{"id":"https://openalex.org/C75684735","wikidata":"https://www.wikidata.org/wiki/Q858810","display_name":"Big data","level":2,"score":0.3555999994277954},{"id":"https://openalex.org/C199683683","wikidata":"https://www.wikidata.org/wiki/Q8799","display_name":"Terabyte","level":2,"score":0.2919999957084656},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.2782999873161316},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.2712000012397766},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2678999900817871},{"id":"https://openalex.org/C11644782","wikidata":"https://www.wikidata.org/wiki/Q15401790","display_name":"Cost efficiency","level":2,"score":0.2662000060081482},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.2540000081062317},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.25189998745918274}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1186/s40537-025-01330-3","is_oa":true,"landing_page_url":"https://doi.org/10.1186/s40537-025-01330-3","pdf_url":null,"source":{"id":"https://openalex.org/S2737955091","display_name":"Journal Of Big Data","issn_l":"2196-1115","issn":["2196-1115"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Big Data","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:45bfbecf1ddd4b38b608b8f6ba694aed","is_oa":false,"landing_page_url":"https://doaj.org/article/45bfbecf1ddd4b38b608b8f6ba694aed","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Journal of Big Data, Vol 13, Iss 1 (2025)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1186/s40537-025-01330-3","is_oa":true,"landing_page_url":"https://doi.org/10.1186/s40537-025-01330-3","pdf_url":null,"source":{"id":"https://openalex.org/S2737955091","display_name":"Journal Of Big Data","issn_l":"2196-1115","issn":["2196-1115"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Big Data","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":55,"referenced_works":["https://openalex.org/W3036444252","https://openalex.org/W3201057442","https://openalex.org/W3208297585","https://openalex.org/W4234833201","https://openalex.org/W4322759267","https://openalex.org/W4324057834","https://openalex.org/W4380885974","https://openalex.org/W4384561707","https://openalex.org/W4385245566","https://openalex.org/W4388725733","https://openalex.org/W4390858893","https://openalex.org/W4390858926","https://openalex.org/W4391631327","https://openalex.org/W4391855109","https://openalex.org/W4393147495","https://openalex.org/W4394782456","https://openalex.org/W4395111253","https://openalex.org/W4396700714","https://openalex.org/W4398238904","https://openalex.org/W4399290596","https://openalex.org/W4399597390","https://openalex.org/W4400324908","https://openalex.org/W4400406431","https://openalex.org/W4401183950","https://openalex.org/W4401612818","https://openalex.org/W4401671778","https://openalex.org/W4402741322","https://openalex.org/W4403401276","https://openalex.org/W4403700226","https://openalex.org/W4403790903","https://openalex.org/W4404781093","https://openalex.org/W4404782610","https://openalex.org/W4404792840","https://openalex.org/W4405655184","https://openalex.org/W4405965505","https://openalex.org/W4406044112","https://openalex.org/W4406366945","https://openalex.org/W4406936679","https://openalex.org/W4407235431","https://openalex.org/W4407243808","https://openalex.org/W4408437773","https://openalex.org/W4408666997","https://openalex.org/W4408876550","https://openalex.org/W4409260121","https://openalex.org/W4409918200","https://openalex.org/W4409980551","https://openalex.org/W4410768311","https://openalex.org/W4410806537","https://openalex.org/W4411696871","https://openalex.org/W4412725690","https://openalex.org/W4412827468","https://openalex.org/W4414281281","https://openalex.org/W4414382167","https://openalex.org/W4414908548","https://openalex.org/W4416025303"],"related_works":[],"abstract_inverted_index":{"The":[0,178],"rapid":[1],"evolution":[2],"of":[3,22],"large":[4],"language":[5],"models":[6],"(LLMs),":[7],"GPT-4":[8],"Turbo,":[9],"Google":[10],"Gemini,":[11],"Qwen,":[12],"Meta\u2019s":[13],"LLaMA":[14,139],"3.1,":[15],"and":[16,41,50,69,83,99,117,138,156,185,194],"DeepSeek-R1":[17,113,149],"has":[18],"redefined":[19],"the":[20,26,142,189],"landscape":[21],"artificial":[23],"intelligence.":[24],"In":[25],"study,":[27],"we":[28],"conduct":[29],"a":[30],"hybrid":[31],"meta-analysis":[32],"integrating":[33],"publicly":[34],"available":[35],"benchmarks,":[36],"model":[37],"cards,":[38],"technical":[39],"reports,":[40],"open-source":[42,145],"repositories":[43],"to":[44,105],"evaluate":[45],"LLMs":[46],"across":[47],"both":[48],"performance":[49],"operational":[51],"dimensions.":[52],"Quantitative":[53],"data":[54],"were":[55],"aggregated":[56],"from":[57],"standardized":[58],"tasks":[59],"such":[60],"as":[61],"MMLU":[62],"(reasoning),":[63],"HumanEval":[64,155],"(code":[65],"generation),":[66],"FLORES-200":[67],"(translation),":[68],"TyDiQA":[70],"(multilingual":[71],"Q&A),":[72],"complemented":[73],"by":[74],"efficiency":[75,102],"metrics":[76],"including":[77],"FLOPs,":[78],"GPU":[79],"hours,":[80],"inference":[81],"latency,":[82],"subscription":[84],"costs.":[85],"A":[86],"big":[87],"data\u2013driven":[88],"KPI":[89],"framework":[90],"covering":[91],"scalability":[92],"index,":[93],"data-throughput":[94],"rate,":[95],"energy":[96],"per":[97],"token,":[98],"training":[100],"cost":[101,186],"was":[103],"applied":[104],"enable":[106],"normalized,":[107],"cross-model":[108],"comparison.":[109],"Results":[110],"indicate":[111],"that":[112],"demonstrates":[114],"strong":[115],"coding":[116],"multilingual":[118],"efficiency,":[119,187],"ChatGPT-4":[120,164],"Turbo":[121],"leads":[122],"in":[123,129,135],"reasoning":[124],"accuracy,":[125,183],"Gemini":[126],"Ultra":[127],"excels":[128],"multimodal":[130,195],"inference,":[131],"Qwen":[132],"is":[133],"competitive":[134],"Chinese-language":[136],"tasks,":[137],"3.1":[140],"remains":[141],"most":[143],"adaptable":[144],"option.":[146],"Across":[147],"datasets,":[148],"achieved":[150],"80.2":[151],"\u00b1":[152,158,167],"1.5%":[153],"on":[154,160],"78.5":[157],"1.8%":[159],"MMLU,":[161],"compared":[162],"with":[163],"Turbo\u2019s":[165],"86.5":[166],"1.9%;":[168],"these":[169],"gaps":[170],"fall":[171],"within":[172],"observed":[173],"heterogeneity":[174],"(I2":[175],"=":[176],"14.6%).":[177],"findings":[179],"highlight":[180],"trade-offs":[181],"among":[182],"scalability,":[184],"emphasizing":[188],"need":[190],"for":[191],"transparent,":[192],"sustainable,":[193],"LLM":[196],"development.":[197]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-05-06T08:25:59.206177","created_date":"2025-12-19T00:00:00"}