{"id":"https://openalex.org/W4402236343","doi":"https://doi.org/10.1007/s11023-024-09695-9","title":"fl-IRT-ing with Psychometrics to Improve NLP Bias Measurement","display_name":"fl-IRT-ing with Psychometrics to Improve NLP Bias Measurement","publication_year":2024,"publication_date":"2024-09-04","ids":{"openalex":"https://openalex.org/W4402236343","doi":"https://doi.org/10.1007/s11023-024-09695-9"},"language":"en","primary_location":{"id":"doi:10.1007/s11023-024-09695-9","is_oa":true,"landing_page_url":"http://dx.doi.org/10.1007/s11023-024-09695-9","pdf_url":"https://link.springer.com/content/pdf/10.1007/s11023-024-09695-9.pdf","source":{"id":"https://openalex.org/S160466889","display_name":"Minds and Machines","issn_l":"0924-6495","issn":["0924-6495","1572-8641"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Minds and Machines","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://link.springer.com/content/pdf/10.1007/s11023-024-09695-9.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5030045811","display_name":"Dominik Bachmann","orcid":"https://orcid.org/0000-0001-5781-6009"},"institutions":[{"id":"https://openalex.org/I193662353","display_name":"Utrecht University","ror":"https://ror.org/04pp8hn57","country_code":"NL","type":"education","lineage":["https://openalex.org/I193662353"]},{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":true,"raw_author_name":"Dominik Bachmann","raw_affiliation_strings":["Department of Experimental Psychology, Utrecht University, Utrecht, The Netherlands","Institute for Logic, Language and Computation, University of Amsterdam, Amsterdam, The Netherlands"],"affiliations":[{"raw_affiliation_string":"Department of Experimental Psychology, Utrecht University, Utrecht, The Netherlands","institution_ids":["https://openalex.org/I193662353"]},{"raw_affiliation_string":"Institute for Logic, Language and Computation, University of Amsterdam, Amsterdam, The Netherlands","institution_ids":["https://openalex.org/I887064364"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058013252","display_name":"Oskar van der Wal","orcid":"https://orcid.org/0000-0003-4087-3714"},"institutions":[{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Oskar van der Wal","raw_affiliation_strings":["Institute for Logic, Language and Computation, University of Amsterdam, Amsterdam, The Netherlands"],"affiliations":[{"raw_affiliation_string":"Institute for Logic, Language and Computation, University of Amsterdam, Amsterdam, The Netherlands","institution_ids":["https://openalex.org/I887064364"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039265561","display_name":"Edita Chvojka","orcid":"https://orcid.org/0000-0002-9909-8276"},"institutions":[{"id":"https://openalex.org/I193662353","display_name":"Utrecht University","ror":"https://ror.org/04pp8hn57","country_code":"NL","type":"education","lineage":["https://openalex.org/I193662353"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Edita Chvojka","raw_affiliation_strings":["Department of Interdisciplinary Social Science, Utrecht University, Utrecht, The Netherlands","Department of Methodology and Statistics, Utrecht University, Utrecht, The Netherlands"],"affiliations":[{"raw_affiliation_string":"Department of Interdisciplinary Social Science, Utrecht University, Utrecht, The Netherlands","institution_ids":["https://openalex.org/I193662353"]},{"raw_affiliation_string":"Department of Methodology and Statistics, Utrecht University, Utrecht, The Netherlands","institution_ids":["https://openalex.org/I193662353"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007928903","display_name":"Willem Zuidema","orcid":"https://orcid.org/0000-0002-2362-5447"},"institutions":[{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Willem H. Zuidema","raw_affiliation_strings":["Institute for Logic, Language and Computation, University of Amsterdam, Amsterdam, The Netherlands"],"affiliations":[{"raw_affiliation_string":"Institute for Logic, Language and Computation, University of Amsterdam, Amsterdam, The Netherlands","institution_ids":["https://openalex.org/I887064364"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041503675","display_name":"Leendert van Maanen","orcid":"https://orcid.org/0000-0001-9120-1075"},"institutions":[{"id":"https://openalex.org/I193662353","display_name":"Utrecht University","ror":"https://ror.org/04pp8hn57","country_code":"NL","type":"education","lineage":["https://openalex.org/I193662353"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Leendert van Maanen","raw_affiliation_strings":["Department of Experimental Psychology, Utrecht University, Utrecht, The Netherlands"],"affiliations":[{"raw_affiliation_string":"Department of Experimental Psychology, Utrecht University, Utrecht, The Netherlands","institution_ids":["https://openalex.org/I193662353"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5081086000","display_name":"Katrin Schulz","orcid":"https://orcid.org/0000-0001-6427-0948"},"institutions":[{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Katrin Schulz","raw_affiliation_strings":["Institute for Logic, Language and Computation, University of Amsterdam, Amsterdam, The Netherlands"],"affiliations":[{"raw_affiliation_string":"Institute for Logic, Language and Computation, University of Amsterdam, Amsterdam, The Netherlands","institution_ids":["https://openalex.org/I887064364"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5030045811"],"corresponding_institution_ids":["https://openalex.org/I193662353","https://openalex.org/I887064364"],"apc_list":{"value":2390,"currency":"EUR","value_usd":2990},"apc_paid":{"value":2390,"currency":"EUR","value_usd":2990},"fwci":0.3407,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":{"value":0.65712489,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":96,"max":98},"biblio":{"volume":"34","issue":"4","first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.996399998664856,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.996399998664856,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9911999702453613,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9634000062942505,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/item-response-theory","display_name":"Item response theory","score":0.6275655031204224},{"id":"https://openalex.org/keywords/psychometrics","display_name":"Psychometrics","score":0.6088968515396118},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5618619322776794},{"id":"https://openalex.org/keywords/philosophy-of-mind","display_name":"Philosophy of mind","score":0.4758738577365875},{"id":"https://openalex.org/keywords/philosophy-of-science","display_name":"Philosophy of science","score":0.4539191722869873},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4441046714782715},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.4411909580230713},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.34662121534347534},{"id":"https://openalex.org/keywords/epistemology","display_name":"Epistemology","score":0.15965884923934937},{"id":"https://openalex.org/keywords/clinical-psychology","display_name":"Clinical psychology","score":0.15389123558998108},{"id":"https://openalex.org/keywords/philosophy","display_name":"Philosophy","score":0.1356692910194397}],"concepts":[{"id":"https://openalex.org/C19875794","wikidata":"https://www.wikidata.org/wiki/Q1207340","display_name":"Item response theory","level":3,"score":0.6275655031204224},{"id":"https://openalex.org/C171606756","wikidata":"https://www.wikidata.org/wiki/Q506132","display_name":"Psychometrics","level":2,"score":0.6088968515396118},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5618619322776794},{"id":"https://openalex.org/C24726157","wikidata":"https://www.wikidata.org/wiki/Q23407","display_name":"Philosophy of mind","level":3,"score":0.4758738577365875},{"id":"https://openalex.org/C110099512","wikidata":"https://www.wikidata.org/wiki/Q59115","display_name":"Philosophy of science","level":2,"score":0.4539191722869873},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4441046714782715},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4411909580230713},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.34662121534347534},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.15965884923934937},{"id":"https://openalex.org/C70410870","wikidata":"https://www.wikidata.org/wiki/Q199906","display_name":"Clinical psychology","level":1,"score":0.15389123558998108},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.1356692910194397},{"id":"https://openalex.org/C182744844","wikidata":"https://www.wikidata.org/wiki/Q35277","display_name":"Metaphysics","level":2,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1007/s11023-024-09695-9","is_oa":true,"landing_page_url":"http://dx.doi.org/10.1007/s11023-024-09695-9","pdf_url":"https://link.springer.com/content/pdf/10.1007/s11023-024-09695-9.pdf","source":{"id":"https://openalex.org/S160466889","display_name":"Minds and Machines","issn_l":"0924-6495","issn":["0924-6495","1572-8641"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Minds and Machines","raw_type":"journal-article"},{"id":"pmh:oai:dare.uva.nl:openaire/1b574aea-55c3-40c0-b4fe-cbb63a510074","is_oa":true,"landing_page_url":"https://dare.uva.nl/personal/pure/en/publications/flirting-with-psychometrics-to-improve-nlp-bias-measurement(1b574aea-55c3-40c0-b4fe-cbb63a510074).html","pdf_url":null,"source":{"id":"https://openalex.org/S4306400088","display_name":"UvA-DARE (University of Amsterdam)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I887064364","host_organization_name":"University of Amsterdam","host_organization_lineage":["https://openalex.org/I887064364"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Bachmann, D, van der Wal, O, Chvojka, E, Zuidema, W H, van Maanen, L & Schulz, K 2024, 'fl-IRT-ing with Psychometrics to Improve NLP Bias Measurement', Minds and Machines, vol. 34, no. 4, 37. https://doi.org/10.1007/s11023-024-09695-9","raw_type":"info:eu-repo/semantics/publishedVersion"},{"id":"pmh:oai:dspace.library.uu.nl:1874/472971","is_oa":true,"landing_page_url":"https://dspace.library.uu.nl/handle/1874/472971","pdf_url":null,"source":{"id":"https://openalex.org/S4306401649","display_name":"Utrecht University Repository (Utrecht University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I193662353","host_organization_name":"Utrecht University","host_organization_lineage":["https://openalex.org/I193662353"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"}],"best_oa_location":{"id":"doi:10.1007/s11023-024-09695-9","is_oa":true,"landing_page_url":"http://dx.doi.org/10.1007/s11023-024-09695-9","pdf_url":"https://link.springer.com/content/pdf/10.1007/s11023-024-09695-9.pdf","source":{"id":"https://openalex.org/S160466889","display_name":"Minds and Machines","issn_l":"0924-6495","issn":["0924-6495","1572-8641"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Minds and Machines","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G309387438","display_name":"The biased reality of online media - \n\nUsing stereotypes to make media manipulation visible","funder_award_id":"406.DI.19.059","funder_id":"https://openalex.org/F4320321800","funder_display_name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek"},{"id":"https://openalex.org/G6009445997","display_name":null,"funder_award_id":"unknown","funder_id":"https://openalex.org/F4320321800","funder_display_name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek"},{"id":"https://openalex.org/G629491556","display_name":null,"funder_award_id":"(NWO)","funder_id":"https://openalex.org/F4320321800","funder_display_name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek"},{"id":"https://openalex.org/G8238660213","display_name":null,"funder_award_id":"06.DI.19.059","funder_id":"https://openalex.org/F4320321800","funder_display_name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek"}],"funders":[{"id":"https://openalex.org/F4320321800","display_name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek","ror":"https://ror.org/04jsz6e67"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4402236343.pdf","grobid_xml":"https://content.openalex.org/works/W4402236343.grobid-xml"},"referenced_works_count":53,"referenced_works":["https://openalex.org/W615644768","https://openalex.org/W1496437928","https://openalex.org/W1535520578","https://openalex.org/W1919216659","https://openalex.org/W2051985103","https://openalex.org/W2160116727","https://openalex.org/W2186921446","https://openalex.org/W2189569146","https://openalex.org/W2264175852","https://openalex.org/W2300094618","https://openalex.org/W2489543173","https://openalex.org/W2561772233","https://openalex.org/W2567418595","https://openalex.org/W2789891878","https://openalex.org/W2913842326","https://openalex.org/W2949678053","https://openalex.org/W2952328691","https://openalex.org/W2963457723","https://openalex.org/W2963526187","https://openalex.org/W2963677523","https://openalex.org/W2979344762","https://openalex.org/W2995006168","https://openalex.org/W2995411104","https://openalex.org/W3034422725","https://openalex.org/W3037831233","https://openalex.org/W3091832279","https://openalex.org/W3092435551","https://openalex.org/W3098371923","https://openalex.org/W3105882417","https://openalex.org/W3115616618","https://openalex.org/W3123340107","https://openalex.org/W3124123577","https://openalex.org/W3133874049","https://openalex.org/W3135734416","https://openalex.org/W3172415559","https://openalex.org/W3175204434","https://openalex.org/W3176477796","https://openalex.org/W3177189402","https://openalex.org/W3177468934","https://openalex.org/W3197577761","https://openalex.org/W3198920343","https://openalex.org/W3207316473","https://openalex.org/W4210622823","https://openalex.org/W4229376125","https://openalex.org/W4234661924","https://openalex.org/W4254972346","https://openalex.org/W4285144733","https://openalex.org/W4285182408","https://openalex.org/W4376647594","https://openalex.org/W4386566763","https://openalex.org/W4390718108","https://openalex.org/W6629567011","https://openalex.org/W6768905222"],"related_works":["https://openalex.org/W2531287006","https://openalex.org/W4220673763","https://openalex.org/W2037107578","https://openalex.org/W2078814290","https://openalex.org/W1965227077","https://openalex.org/W2052861746","https://openalex.org/W2043402388","https://openalex.org/W2100605390","https://openalex.org/W2953698770","https://openalex.org/W109703496"],"abstract_inverted_index":{"Abstract":[0],"To":[1],"prevent":[2],"ordinary":[3],"people":[4],"from":[5,79,146],"being":[6],"harmed":[7],"by":[8,193],"natural":[9],"language":[10,23,57,149,212,237],"processing":[11],"(NLP)":[12],"technology,":[13],"finding":[14],"ways":[15],"to":[16,20,54,93,108,138,161,167,203,218],"measure":[17],"the":[18,80,109,128,136,159,163,183,224],"extent":[19,137,160],"which":[21,139,162],"a":[22,56,100,156,236,240],"model":[24,177,213,238],"is":[25],"biased":[26,145,148,241],"(e.g.,":[27,135,158],"regarding":[28],"gender)":[29],"has":[30],"become":[31],"an":[32,91,140],"active":[33],"area":[34],"of":[35,40,48,83,102,114,130,176,185],"research.":[36],"One":[37],"popular":[38],"class":[39],"NLP":[41],"bias":[42,45,72,115,214],"measures":[43],"are":[44,52,215],"benchmark":[46,116,164,186],"datasets\u2014collections":[47],"test":[49,133],"items":[50,134],"that":[51,70,207],"meant":[53],"assess":[55,168],"model\u2019s":[58],"preference":[59],"for":[60,126,191,211],"stereotypical":[61],"versus":[62],"non-stereotypical":[63],"language.":[64],"In":[65],"this":[66],"paper,":[67],"we":[68,89,205],"argue":[69,206],"such":[71,180],"benchmarks":[73,154],"should":[74],"be":[75,106,189],"assessed":[76],"with":[77,99,123],"models":[78,98],"psychometric":[81],"framework":[82],"item":[84,141],"response":[85],"theory":[86],"(IRT).":[87],"Specifically,":[88],"tie":[90],"introduction":[92],"basic":[94],"IRT":[95,120,208],"concepts":[96],"and":[97,112],"discussion":[101],"how":[103],"they":[104],"could":[105,188],"relevant":[107],"evaluation,":[110,119],"interpretation":[111],"improvement":[113],"datasets.":[117],"Regarding":[118],"provides":[121],"us":[122,166],"methodological":[124],"tools":[125],"assessing":[127],"quality":[129,184],"both":[131],"individual":[132],"can":[142],"differentiate":[143],"highly":[144],"less":[147],"models)":[150],"as":[151,153,155,223],"well":[152],"whole":[157],"allows":[165],"not":[169,235],"only":[170],"severe":[171],"but":[172],"also":[173],"subtle":[174],"levels":[175],"bias).":[178],"Through":[179],"diagnostic":[181],"tools,":[182],"datasets":[187],"improved,":[190],"example":[192],"deleting":[194],"or":[195,234],"reworking":[196],"poorly":[197],"performing":[198],"items.":[199],"Finally,":[200],"in":[201],"regards":[202],"interpretation,":[204],"models\u2019":[209],"estimates":[210],"conceptually":[216],"superior":[217],"traditional":[219],"accuracy-based":[220],"evaluation":[221],"metrics,":[222],"former":[225],"take":[226],"into":[227],"account":[228],"more":[229],"information":[230],"than":[231],"just":[232],"whether":[233],"provided":[239],"response.":[242]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-04-10T15:06:20.359241","created_date":"2025-10-10T00:00:00"}