{"id":"https://openalex.org/W7225384351","doi":"https://doi.org/10.1017/nlp.2026.10046","title":"When AI judges AI: Who defines quality in responsible NLP?","display_name":"When AI judges AI: Who defines quality in responsible NLP?","publication_year":2026,"publication_date":"2026-11-06","ids":{"openalex":"https://openalex.org/W7225384351","doi":"https://doi.org/10.1017/nlp.2026.10046"},"language":"en","primary_location":{"id":"doi:10.1017/nlp.2026.10046","doi":"https://doi.org/10.1017/nlp.2026.10046","is_oa":true,"landing_page_url":"https://doi.org/10.1017/nlp.2026.10046","pdf_url":"https://www.cambridge.org/core/services/aop-cambridge-core/content/view/67D7F9EF518F7668109640A6C9D47C92/S2977042426100466a.pdf/div-class-title-when-ai-judges-ai-who-defines-quality-in-responsible-nlp-div.pdf","source":{"id":"https://openalex.org/S4404676627","display_name":"Natural language processing.","issn_l":"2977-0424","issn":["2977-0424"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["ccf-c","dongbi-c","erih-plus","russia-white-list-1"],"host_organization":"https://openalex.org/P4310311721","host_organization_name":"Cambridge University Press","host_organization_lineage":["https://openalex.org/P4310311721","https://openalex.org/P4310311702"],"host_organization_lineage_names":["Cambridge University Press","University of Cambridge"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Natural Language Processing","raw_type":"journal-article"},"volume":null,"type":"article","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://www.cambridge.org/core/services/aop-cambridge-core/content/view/67D7F9EF518F7668109640A6C9D47C92/S2977042426100466a.pdf/div-class-title-when-ai-judges-ai-who-defines-quality-in-responsible-nlp-div.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5070022854","display_name":"Salima Lamsiyah","orcid":"https://orcid.org/0000-0001-8789-5713","observed_orcids":["https://orcid.org/0000-0001-8789-5713"]},"institutions":[{"id":"https://openalex.org/I186903577","display_name":"University of Luxembourg","ror":"https://ror.org/036x5ad56","country_code":"LU","type":"education","lineage":["https://openalex.org/I186903577"]}],"countries":["LU"],"is_corresponding":true,"raw_author_name":"Salima Lamsiyah","raw_affiliation_strings":["University of Luxembourg"],"raw_orcid":"https://orcid.org/0000-0001-8789-5713","affiliations":[{"raw_affiliation_string":"University of Luxembourg","institution_ids":["https://openalex.org/I186903577"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5070022854"],"corresponding_institution_ids":["https://openalex.org/I186903577"],"apc_list":{"value":3550,"currency":"USD","value_usd":3550},"apc_paid":{"value":3550,"currency":"USD","value_usd":3550},"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.0,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"11"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9854000210762024,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9854000210762024,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.0024999999441206455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.002099999925121665,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/automated-assessment","display_name":"automated assessment","score":0.7099999785423279},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.9169999957084656},{"id":"https://openalex.org/keywords/epistemic-injustice","display_name":"epistemic injustice","score":0.8899999856948853},{"id":"https://openalex.org/keywords/measurement-theory","display_name":"measurement theory","score":0.7770000100135803},{"id":"https://openalex.org/keywords/benchmarking","display_name":"benchmarking","score":0.37700000405311584},{"id":"https://openalex.org/keywords/training-sample-selection","display_name":"training data selection","score":0.6650000214576721},{"id":"https://openalex.org/keywords/reliability-and-validity","display_name":"reliability and validity","score":0.6060000061988831},{"id":"https://openalex.org/keywords/reasonable-disagreement","display_name":"reasonable disagreement","score":0.5099999904632568},{"id":"https://openalex.org/keywords/accountability","display_name":"accountability","score":0.3019999861717224},{"id":"https://openalex.org/keywords/distributed-responsibility","display_name":"distributed responsibility","score":0.33399999141693115}],"concepts":[{"id":"https://openalex.org/C111640148","wikidata":"https://www.wikidata.org/wiki/Q847349","display_name":"Rubric","level":2,"score":0.7282999753952026},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.6873000264167786},{"id":"https://openalex.org/C2776916960","wikidata":"https://www.wikidata.org/wiki/Q637156","display_name":"Warrant","level":2,"score":0.6524999737739563},{"id":"https://openalex.org/C46295352","wikidata":"https://www.wikidata.org/wiki/Q207982","display_name":"Legitimacy","level":3,"score":0.6050999760627747},{"id":"https://openalex.org/C98184364","wikidata":"https://www.wikidata.org/wiki/Q1780131","display_name":"Argument (complex analysis)","level":2,"score":0.5856999754905701},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.48809999227523804},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.3871999979019165},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.362199991941452},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.35569998621940613},{"id":"https://openalex.org/C2780224610","wikidata":"https://www.wikidata.org/wiki/Q1530061","display_name":"Credibility","level":2,"score":0.34599998593330383},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.3447999954223633},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3391999900341034},{"id":"https://openalex.org/C153701036","wikidata":"https://www.wikidata.org/wiki/Q659974","display_name":"Trustworthiness","level":2,"score":0.3278000056743622},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.32589998841285706},{"id":"https://openalex.org/C127705205","wikidata":"https://www.wikidata.org/wiki/Q5748245","display_name":"Heuristics","level":2,"score":0.31690001487731934},{"id":"https://openalex.org/C100609095","wikidata":"https://www.wikidata.org/wiki/Q1335050","display_name":"Embodied cognition","level":2,"score":0.31439998745918274},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.29120001196861267},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.2897999882698059},{"id":"https://openalex.org/C7991579","wikidata":"https://www.wikidata.org/wiki/Q17955","display_name":"Criticism","level":2,"score":0.28949999809265137},{"id":"https://openalex.org/C197947376","wikidata":"https://www.wikidata.org/wiki/Q5155608","display_name":"Comparability","level":2,"score":0.2867000102996826},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.28349998593330383},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.2676999866962433},{"id":"https://openalex.org/C2776548248","wikidata":"https://www.wikidata.org/wiki/Q12621536","display_name":"Judgement","level":2,"score":0.2515999972820282}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1017/nlp.2026.10046","doi":"https://doi.org/10.1017/nlp.2026.10046","is_oa":true,"landing_page_url":"https://doi.org/10.1017/nlp.2026.10046","pdf_url":"https://www.cambridge.org/core/services/aop-cambridge-core/content/view/67D7F9EF518F7668109640A6C9D47C92/S2977042426100466a.pdf/div-class-title-when-ai-judges-ai-who-defines-quality-in-responsible-nlp-div.pdf","source":{"id":"https://openalex.org/S4404676627","display_name":"Natural language processing.","issn_l":"2977-0424","issn":["2977-0424"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["ccf-c","dongbi-c","erih-plus","russia-white-list-1"],"host_organization":"https://openalex.org/P4310311721","host_organization_name":"Cambridge University Press","host_organization_lineage":["https://openalex.org/P4310311721","https://openalex.org/P4310311702"],"host_organization_lineage_names":["Cambridge University Press","University of Cambridge"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Natural Language Processing","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1017/nlp.2026.10046","doi":"https://doi.org/10.1017/nlp.2026.10046","is_oa":true,"landing_page_url":"https://doi.org/10.1017/nlp.2026.10046","pdf_url":"https://www.cambridge.org/core/services/aop-cambridge-core/content/view/67D7F9EF518F7668109640A6C9D47C92/S2977042426100466a.pdf/div-class-title-when-ai-judges-ai-who-defines-quality-in-responsible-nlp-div.pdf","source":{"id":"https://openalex.org/S4404676627","display_name":"Natural language processing.","issn_l":"2977-0424","issn":["2977-0424"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["ccf-c","dongbi-c","erih-plus","russia-white-list-1"],"host_organization":"https://openalex.org/P4310311721","host_organization_name":"Cambridge University Press","host_organization_lineage":["https://openalex.org/P4310311721","https://openalex.org/P4310311702"],"host_organization_lineage_names":["Cambridge University Press","University of Cambridge"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Natural Language Processing","raw_type":"journal-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7225384351.pdf","grobid_xml":"https://content.openalex.org/works/W7225384351.grobid-xml"},"referenced_works_count":22,"referenced_works":["https://openalex.org/W2028829385","https://openalex.org/W2897042519","https://openalex.org/W2911227954","https://openalex.org/W2995006168","https://openalex.org/W3001807593","https://openalex.org/W3032388710","https://openalex.org/W3038795242","https://openalex.org/W4252633962","https://openalex.org/W4385573849","https://openalex.org/W4389519254","https://openalex.org/W4404782847","https://openalex.org/W4404783034","https://openalex.org/W4412889486","https://openalex.org/W4416034673","https://openalex.org/W4416037109","https://openalex.org/W7120088840","https://openalex.org/W7133239970","https://openalex.org/W7160631968","https://openalex.org/W7166531721","https://openalex.org/W7166871087","https://openalex.org/W7204221468","https://openalex.org/W7213967696"],"related_works":["https://openalex.org/W7222922938","https://openalex.org/W7223106854","https://openalex.org/W7160985808","https://openalex.org/W4412944552","https://openalex.org/W7168035120","https://openalex.org/W7140209630","https://openalex.org/W6966851422","https://openalex.org/W7214581315","https://openalex.org/W7105695506","https://openalex.org/W7223992886"],"abstract_inverted_index":{"Abstract":[0],"What":[1],"gives":[2],"an":[3,86,217],"automated":[4,87],"judgment":[5,88,180],"the":[6,18,120,127,176,193,198],"authority":[7,128],"to":[8,92,122,129,182,245],"define":[9],"a":[10,58,61,63,67,103,124,168,237],"good":[11],"answer?":[12],"Large":[13],"language":[14],"models":[15],"increasingly":[16],"evaluate":[17],"outputs":[19],"of":[20,42,178,195,201,219,231],"other":[21],"models,":[22],"helping":[23],"researchers":[24],"compare":[25],"systems,":[26],"select":[27],"training":[28,68],"data,":[29],"and":[30,94,99,112,134,142,157,197,227],"inform":[31,236],"deployment.":[32],"However,":[33],"agreement":[34],"with":[35,228],"human":[36],"judgments":[37,222],"cannot":[38,240],"settle":[39],"whose":[40,148],"understanding":[41],"quality":[43,53],"should":[44,89,161,223],"guide":[45],"these":[46],"decisions.":[47],"Automated":[48],"evaluation":[49,209],"can":[50,72,235],"influence":[51],"how":[52],"itself":[54],"is":[55,150,206],"defined.":[56],"When":[57],"rubric":[59],"becomes":[60],"benchmark,":[62],"selection":[64],"rule,":[65],"or":[66],"signal,":[69],"its":[70,242],"assumptions":[71],"become":[73],"standards":[74],"that":[75,207],"others":[76],"inherit.":[77],"This":[78],"piece":[79],"develops":[80],"bounded":[81],"evaluative":[82],"authority:":[83],"reliance":[84],"on":[85,115,192],"extend":[90],"only":[91],"interpretations":[93],"decisions":[95],"supported":[96],"by":[97],"evidence":[98],"accountable":[100],"reasons":[101],"within":[102],"stated":[104],"scope.":[105],"Drawing":[106],"together":[107],"measurement":[108],"theory,":[109],"evaluator":[110],"research,":[111],"philosophical":[113],"work":[114],"epistemic":[116],"injustice,":[117],"it":[118,215,239],"distinguishes":[119],"ability":[121],"apply":[123],"criterion":[125],"from":[126],"choose":[130],"it.":[131,247],"Reliability,":[132],"validity,":[133],"legitimacy":[135],"address":[136],"connected":[137],"questions":[138],"about":[139],"consistency,":[140],"meaning,":[141],"justified":[143],"reliance.":[144],"The":[145,184],"discussion":[146],"considers":[147],"knowledge":[149],"recognised,":[151],"who":[152],"may":[153],"resolve":[154],"conflicting":[155],"values,":[156],"when":[158],"reasonable":[159],"disagreement":[160],"remain":[162],"visible":[163],"rather":[164],"than":[165,212],"disappear":[166],"into":[167],"single":[169],"score.":[170],"An":[171],"Evaluative":[172],"Authority":[173],"Envelope":[174],"makes":[175],"grounds":[177],"delegated":[179],"open":[181],"scrutiny.":[183],"argument":[185],"extends":[186],"earlier":[187],"pieces":[188],"in":[189],"Responsible":[190],"NLP":[191],"limits":[194],"performance":[196],"institutional":[199],"distribution":[200],"responsibility.":[202],"Its":[203],"central":[204],"claim":[205],"responsible":[208],"requires":[210,216],"more":[211],"accurate":[213],"scoring:":[214],"account":[218],"why":[220],"particular":[221],"count,":[224],"for":[225],"whom,":[226],"what":[229],"possibility":[230],"challenge.":[232],"A":[233],"score":[234],"decision;":[238],"supply":[241],"own":[243],"warrant":[244],"govern":[246]},"counts_by_year":[],"updated_date":"2026-11-08T12:12:28.789561","created_date":"2026-11-07T00:00:00"}