{"id":"https://openalex.org/W7165769925","doi":"https://doi.org/10.48550/arxiv.2606.24596","title":"To Compare, or Not to Compare: On Methodological Practices in Evaluating Social Bias","display_name":"To Compare, or Not to Compare: On Methodological Practices in Evaluating Social Bias","publication_year":2026,"publication_date":"2026-06-23","ids":{"openalex":"https://openalex.org/W7165769925","doi":"https://doi.org/10.48550/arxiv.2606.24596"},"language":"en","primary_location":{"id":"pmh:oai:tubiblio.ulb.tu-darmstadt.de:161143","is_oa":false,"landing_page_url":"http://tubiblio.ulb.tu-darmstadt.de/view/person/Marcuzzi=3AFederico=3A=3A.html>","pdf_url":null,"source":{"id":"https://openalex.org/S4377196390","display_name":"TUbilio (Technical University of Darmstadt)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I31512782","host_organization_name":"Technische Universit\u00e4t Darmstadt","host_organization_lineage":["https://openalex.org/I31512782"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":null,"raw_type":"Report"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.24596","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5009608294","display_name":"Federico Marcuzzi","orcid":"https://orcid.org/0000-0002-8141-8294"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Marcuzzi, Federico","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086217226","display_name":"Xuefei Ning","orcid":"https://orcid.org/0000-0003-2209-8312"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ning, Xuefei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139271239","display_name":"Roy Schwartz","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Schwartz, Roy","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5139224918","display_name":"Iryna Gurevych","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gurevych, Iryna","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.28049999475479126,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.28049999475479126,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.1388999968767166,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.07280000299215317,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/prejudice","display_name":"Prejudice (legal term)","score":0.6223999857902527},{"id":"https://openalex.org/keywords/framing","display_name":"Framing (construction)","score":0.5146999955177307},{"id":"https://openalex.org/keywords/leverage","display_name":"Leverage (statistics)","score":0.4790000021457672},{"id":"https://openalex.org/keywords/contrast","display_name":"Contrast (vision)","score":0.45339998602867126},{"id":"https://openalex.org/keywords/debiasing","display_name":"Debiasing","score":0.41370001435279846},{"id":"https://openalex.org/keywords/audit","display_name":"Audit","score":0.4097999930381775},{"id":"https://openalex.org/keywords/motivated-reasoning","display_name":"Motivated reasoning","score":0.33570000529289246},{"id":"https://openalex.org/keywords/phenomenon","display_name":"Phenomenon","score":0.33550000190734863}],"concepts":[{"id":"https://openalex.org/C107062074","wikidata":"https://www.wikidata.org/wiki/Q109701697","display_name":"Prejudice (legal term)","level":2,"score":0.6223999857902527},{"id":"https://openalex.org/C169087156","wikidata":"https://www.wikidata.org/wiki/Q2131593","display_name":"Framing (construction)","level":2,"score":0.5146999955177307},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.5059999823570251},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.4790000021457672},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.4681999981403351},{"id":"https://openalex.org/C2776502983","wikidata":"https://www.wikidata.org/wiki/Q690182","display_name":"Contrast (vision)","level":2,"score":0.45339998602867126},{"id":"https://openalex.org/C2779458634","wikidata":"https://www.wikidata.org/wiki/Q24963715","display_name":"Debiasing","level":2,"score":0.41370001435279846},{"id":"https://openalex.org/C199521495","wikidata":"https://www.wikidata.org/wiki/Q181487","display_name":"Audit","level":2,"score":0.4097999930381775},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.39480000734329224},{"id":"https://openalex.org/C2776325391","wikidata":"https://www.wikidata.org/wiki/Q6917865","display_name":"Motivated reasoning","level":3,"score":0.33570000529289246},{"id":"https://openalex.org/C50335755","wikidata":"https://www.wikidata.org/wiki/Q483247","display_name":"Phenomenon","level":2,"score":0.33550000190734863},{"id":"https://openalex.org/C2991991027","wikidata":"https://www.wikidata.org/wiki/Q6007314","display_name":"Implicit bias","level":2,"score":0.30799999833106995},{"id":"https://openalex.org/C118084267","wikidata":"https://www.wikidata.org/wiki/Q26110","display_name":"Positive economics","level":1,"score":0.29789999127388},{"id":"https://openalex.org/C108115952","wikidata":"https://www.wikidata.org/wiki/Q5155650","display_name":"Comparative research","level":2,"score":0.2976999878883362},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.29109999537467957},{"id":"https://openalex.org/C94617000","wikidata":"https://www.wikidata.org/wiki/Q639854","display_name":"Qualitative comparative analysis","level":2,"score":0.28790000081062317},{"id":"https://openalex.org/C79585631","wikidata":"https://www.wikidata.org/wiki/Q431498","display_name":"Confirmation bias","level":2,"score":0.28600001335144043},{"id":"https://openalex.org/C136714292","wikidata":"https://www.wikidata.org/wiki/Q1440683","display_name":"Framing effect","level":3,"score":0.27230000495910645},{"id":"https://openalex.org/C194959371","wikidata":"https://www.wikidata.org/wiki/Q17056235","display_name":"Inequity aversion","level":3,"score":0.27149999141693115},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.2662999927997589},{"id":"https://openalex.org/C189216375","wikidata":"https://www.wikidata.org/wiki/Q1127759","display_name":"Cognitive bias","level":3,"score":0.25929999351501465},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.25130000710487366}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:tubiblio.ulb.tu-darmstadt.de:161143","is_oa":false,"landing_page_url":"http://tubiblio.ulb.tu-darmstadt.de/view/person/Marcuzzi=3AFederico=3A=3A.html>","pdf_url":null,"source":{"id":"https://openalex.org/S4377196390","display_name":"TUbilio (Technical University of Darmstadt)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I31512782","host_organization_name":"Technische Universit\u00e4t Darmstadt","host_organization_lineage":["https://openalex.org/I31512782"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":null,"raw_type":"Report"},{"id":"doi:10.48550/arxiv.2606.24596","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.24596","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.24596","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.24596","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Reduced inequalities","id":"https://metadata.un.org/sdg/10","score":0.6868264675140381}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"As":[0],"Large":[1],"Language":[2],"Models":[3],"are":[4,143],"increasingly":[5],"deployed":[6],"in":[7,84,195],"critical":[8],"applications,":[9],"robustly":[10,184],"evaluating":[11],"their":[12],"social":[13,85,126],"biases":[14,127],"is":[15,160],"paramount.":[16],"However,":[17],"the":[18,35,71],"current":[19],"literature":[20],"suffers":[21],"from":[22,33],"widespread":[23],"methodological":[24,175],"fragmentation,":[25],"which":[26],"yields":[27],"contradictory":[28],"conclusions.":[29],"This":[30],"stems":[31],"largely":[32],"ignoring":[34],"structural":[36,82],"framing":[37],"of":[38,74],"benchmark-level":[39],"evaluations.":[40,87],"To":[41],"resolve":[42],"this,":[43],"we":[44,154,171],"introduce":[45],"a":[46,95,115,137,161,173],"unified":[47],"and":[48,80,131],"controllable":[49],"framework":[50],"that":[51,156,164],"standardizes":[52],"heterogeneous":[53],"benchmarks":[54],"to":[55,69,150,183],"systematically":[56],"contrast":[57],"isolated":[58,101],"demographic":[59],"assessments":[60,102],"with":[61,167],"forced-choice":[62],"comparative":[63,106,129,158,181,193],"settings.":[64],"Crucially,":[65],"this":[66,132,157],"allows":[67],"us":[68],"disentangle":[70],"confounding":[72],"effects":[73],"Chain-of-Thought":[75],"reasoning,":[76],"neutral":[77,145],"fallback":[78,146],"options,":[79],"other":[81],"artifacts":[83],"bias":[86,134],"Our":[88],"evaluation":[89],"across":[90],"multiple":[91],"model":[92,168],"families":[93],"reveals":[94],"massive,":[96],"systematic":[97],"paradigm":[98],"gap:":[99],"while":[100,177],"limit":[103],"prejudice":[104,139,159],"activation,":[105],"settings":[107,182],"act":[108],"as":[109,136],"aggressive":[110],"catalysts":[111],"for":[112],"latent":[113],"discrimination,":[114],"shift":[116],"primarily":[117],"driven":[118],"by":[119],"underspecified":[120],"contexts.":[121],"Alarmingly,":[122],"CoT":[123],"reasoning":[124],"exacerbates":[125],"under":[128],"settings,":[130],"systemic":[133],"persists":[135],"deterministic":[138],"even":[140],"when":[141],"models":[142],"provided":[144],"options":[147],"or":[148],"claim":[149],"answer":[151],"randomly.":[152],"Finally,":[153],"demonstrate":[155],"generalized":[162],"phenomenon":[163],"scales":[165],"positively":[166],"size.":[169],"Ultimately,":[170],"offer":[172],"crucial":[174],"guideline:":[176],"researchers":[178],"must":[179],"leverage":[180],"audit":[185],"hidden":[186],"biases,":[187],"practitioners":[188],"cannot":[189],"safely":[190],"rely":[191],"on":[192],"deployments":[194],"ambiguous":[196],"real-world":[197],"tasks.":[198]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-06-25T00:00:00"}