{"id":"https://openalex.org/W4393318216","doi":"https://doi.org/10.31219/osf.io/mc762","title":"A Comparative Analysis to Evaluate Bias and Fairness Across Large Language Models with Benchmarks","display_name":"A Comparative Analysis to Evaluate Bias and Fairness Across Large Language Models with Benchmarks","publication_year":2024,"publication_date":"2024-03-29","ids":{"openalex":"https://openalex.org/W4393318216","doi":"https://doi.org/10.31219/osf.io/mc762"},"language":"en","primary_location":{"id":"doi:10.31219/osf.io/mc762","doi":"https://doi.org/10.31219/osf.io/mc762","is_oa":true,"landing_page_url":"https://doi.org/10.31219/osf.io/mc762","pdf_url":"https://osf.io/mc762/download","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":null,"raw_type":"posted-content"},"volume":null,"type":"preprint","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://osf.io/mc762/download","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5094221642","display_name":"\u9673\u6587\u610f","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"\u9673\u6587\u610f","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5094221643","display_name":"\u9ec3\u5146\u660e","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"\u9ec3\u5146\u660e","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":19,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.00039999998989515007,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12262","display_name":"Hate Speech and Cyberbullying Detection","score":0.00019999999494757503,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.9779999852180481},{"id":"https://openalex.org/keywords/algorithmic-bias","display_name":"algorithmic bias","score":0.9020000100135803},{"id":"https://openalex.org/keywords/fairness-evaluation","display_name":"fairness evaluation","score":0.7950000166893005},{"id":"https://openalex.org/keywords/chatgpt-4","display_name":"ChatGPT-4","score":0.7789999842643738},{"id":"https://openalex.org/keywords/google-gemini","display_name":"Google Gemini","score":0.9229999780654907},{"id":"https://openalex.org/keywords/llama-2","display_name":"Llama 2","score":0.9900000095367432},{"id":"https://openalex.org/keywords/big-bench","display_name":"BIG-Bench","score":0.8450000286102295},{"id":"https://openalex.org/keywords/gender-bias","display_name":"gender bias","score":0.8809999823570251},{"id":"https://openalex.org/keywords/racial-bias","display_name":"racial bias","score":0.9160000085830688},{"id":"https://openalex.org/keywords/ethnic-prejudice","display_name":"ethnic prejudice","score":0.7350000143051147},{"id":"https://openalex.org/keywords/benchmarking","display_name":"benchmarking","score":0.4880000054836273},{"id":"https://openalex.org/keywords/ai-ethics","display_name":"AI ethics","score":0.7919999957084656}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5175127983093262},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.33978164196014404},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.23323208093643188}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.31219/osf.io/mc762","doi":"https://doi.org/10.31219/osf.io/mc762","is_oa":true,"landing_page_url":"https://doi.org/10.31219/osf.io/mc762","pdf_url":"https://osf.io/mc762/download","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":null,"raw_type":"posted-content"},{"id":"pmh:oai:share.osf.io:67368629-96cb-48dd-9ec4-d0ca819e285a","doi":null,"is_oa":true,"landing_page_url":"https://osf.io/mc762","pdf_url":null,"source":{"id":"https://openalex.org/S4306401127","display_name":"OSF Preprints (OSF Preprints)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2799848540","host_organization_name":"Center for Open Science","host_organization_lineage":["https://openalex.org/I2799848540"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.31219/osf.io/mc762","doi":"https://doi.org/10.31219/osf.io/mc762","is_oa":true,"landing_page_url":"https://doi.org/10.31219/osf.io/mc762","pdf_url":"https://osf.io/mc762/download","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":null,"raw_type":"posted-content"},"sustainable_development_goals":[{"display_name":"Reduced inequalities","id":"https://metadata.un.org/sdg/10","score":0.7578999996185303}],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4393318216.pdf","grobid_xml":"https://content.openalex.org/works/W4393318216.grobid-xml"},"referenced_works_count":62,"referenced_works":["https://openalex.org/W2983486364","https://openalex.org/W3016300649","https://openalex.org/W3019489177","https://openalex.org/W3030163527","https://openalex.org/W3034115845","https://openalex.org/W3035032873","https://openalex.org/W3092298059","https://openalex.org/W3101767999","https://openalex.org/W3128330592","https://openalex.org/W3130450512","https://openalex.org/W3152258780","https://openalex.org/W3158733382","https://openalex.org/W3172205429","https://openalex.org/W3175060421","https://openalex.org/W3182414949","https://openalex.org/W3195577433","https://openalex.org/W4220993274","https://openalex.org/W4221045317","https://openalex.org/W4221167110","https://openalex.org/W4226112058","https://openalex.org/W4226146865","https://openalex.org/W4280650171","https://openalex.org/W4281690148","https://openalex.org/W4285192297","https://openalex.org/W4285199616","https://openalex.org/W4313527987","https://openalex.org/W4316038168","https://openalex.org/W4320468833","https://openalex.org/W4322766882","https://openalex.org/W4324373918","https://openalex.org/W4376643691","https://openalex.org/W4381572755","https://openalex.org/W4382246105","https://openalex.org/W4384071683","https://openalex.org/W4385227045","https://openalex.org/W4385632485","https://openalex.org/W4386566565","https://openalex.org/W4386728881","https://openalex.org/W4387144045","https://openalex.org/W4387617694","https://openalex.org/W4388598844","https://openalex.org/W4388691793","https://openalex.org/W4389043118","https://openalex.org/W4390002645","https://openalex.org/W4390602553","https://openalex.org/W4390833061","https://openalex.org/W4391136348","https://openalex.org/W4391136507","https://openalex.org/W4391855109","https://openalex.org/W4392308773","https://openalex.org/W4392353733","https://openalex.org/W4401131969","https://openalex.org/W4401958727","https://openalex.org/W4404534210","https://openalex.org/W4404782667","https://openalex.org/W4406800520","https://openalex.org/W4410342726","https://openalex.org/W4415725257","https://openalex.org/W7011884576","https://openalex.org/W7133207027","https://openalex.org/W7133226801","https://openalex.org/W7133238093"],"related_works":["https://openalex.org/W4410781953","https://openalex.org/W4407690681","https://openalex.org/W7129205257","https://openalex.org/W4405395068","https://openalex.org/W4389611182","https://openalex.org/W4415398275","https://openalex.org/W4396698999","https://openalex.org/W4393407165","https://openalex.org/W4415430933","https://openalex.org/W4405465435"],"abstract_inverted_index":{"This":[0],"study":[1,102],"performs":[2],"a":[3,134,156],"comprehensive":[4],"evaluation":[5],"of":[6,32,70,151,174],"bias":[7,71,99,126],"and":[8,19,46,98,111,119,164,172],"fairness":[9,97,171],"within":[10],"Large":[11],"Language":[12],"Models":[13],"(LLMs),":[14],"including":[15],"ChatGPT-4,":[16],"Google":[17,24,49],"Gemini,":[18],"Llama":[20],"2,":[21],"utilizing":[22],"the":[23,67,104,114,148,170],"BIG-Bench":[25,50],"benchmark.":[26],"Our":[27],"analysis":[28,79],"reveals":[29],"varied":[30],"levels":[31],"biases":[33],"across":[34,95],"models,":[35],"with":[36],"disparities":[37],"particularly":[38],"notable":[39],"in":[40,54,65,74,87,121,161],"dimensions":[41],"such":[42],"as":[43],"gender,":[44],"race,":[45],"ethnicity.":[47],"The":[48,101,153],"benchmark":[51],"proved":[52],"instrumental":[53],"identifying":[55],"these":[56],"biases,":[57],"though":[58],"its":[59],"effectiveness":[60],"is":[61],"tempered":[62],"by":[63],"challenges":[64],"capturing":[66],"sophisticated":[68],"manifestations":[69],"that":[72,81],"emerge":[73],"real-world":[75],"contexts.":[76],"Comparative":[77],"performance":[78],"indicates":[80],"while":[82],"each":[83],"model":[84,92,108,162],"exhibits":[85],"strengths":[86],"certain":[88],"areas,":[89],"no":[90],"single":[91],"uniformly":[93],"excels":[94],"all":[96],"metrics.":[100],"underscores":[103],"intricate":[105],"balance":[106],"between":[107],"performance,":[109],"fairness,":[110],"efficiency,":[112],"highlighting":[113],"necessity":[115],"for":[116,133,158],"ongoing":[117],"research":[118,131],"development":[120],"AI":[122,138],"ethics":[123],"to":[124,137,146,168],"mitigate":[125],"effectively.":[127],"Insights":[128],"from":[129],"this":[130],"advocate":[132],"multifaceted":[135],"approach":[136],"development,":[139],"integrating":[140],"ethical":[141],"considerations":[142],"at":[143],"every":[144],"stage":[145],"ensure":[147],"equitable":[149],"advancement":[150],"technology.":[152],"findings":[154],"prompt":[155],"call":[157],"continued":[159],"innovation":[160],"training":[163],"benchmarking":[165],"methodologies,":[166],"aiming":[167],"enhance":[169],"inclusivity":[173],"future":[175],"LLMs.":[176]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":18}],"updated_date":"2026-10-29T09:56:32.521200","created_date":"2024-03-30T00:00:00"}