{"id":"https://openalex.org/W7147729230","doi":"https://doi.org/10.48550/arxiv.2603.28351","title":"Not All Subjectivity Is the Same! Defining Desiderata for the Evaluation of Subjectivity in NLP","display_name":"Not All Subjectivity Is the Same! Defining Desiderata for the Evaluation of Subjectivity in NLP","publication_year":2026,"publication_date":"2026-03-30","ids":{"openalex":"https://openalex.org/W7147729230","doi":"https://doi.org/10.48550/arxiv.2603.28351"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.28351","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.28351","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.28351","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5021578462","display_name":"Urja Khurana","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Khurana, Urja","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5132709632","display_name":"Michiel van der Meer","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"van der Meer, Michiel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5073706887","display_name":"Enrico Liscio","orcid":"https://orcid.org/0000-0002-8285-5867"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liscio, Enrico","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074075557","display_name":"Antske Fokkens","orcid":"https://orcid.org/0000-0002-6628-6916"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fokkens, Antske","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5039587200","display_name":"Pradeep K. Murukannaiah","orcid":"https://orcid.org/0000-0002-1261-6908"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Murukannaiah, Pradeep K.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.2484000027179718,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.2484000027179718,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12262","display_name":"Hate Speech and Cyberbullying Detection","score":0.12359999865293503,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.08540000021457672,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/subjectivity","display_name":"Subjectivity","score":0.9531999826431274},{"id":"https://openalex.org/keywords/polyphony","display_name":"Polyphony","score":0.6442000269889832},{"id":"https://openalex.org/keywords/diversity","display_name":"Diversity (politics)","score":0.36500000953674316},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.34599998593330383},{"id":"https://openalex.org/keywords/realization","display_name":"Realization (probability)","score":0.2809999883174896}],"concepts":[{"id":"https://openalex.org/C202889954","wikidata":"https://www.wikidata.org/wiki/Q1139554","display_name":"Subjectivity","level":2,"score":0.9531999826431274},{"id":"https://openalex.org/C128979739","wikidata":"https://www.wikidata.org/wiki/Q179465","display_name":"Polyphony","level":2,"score":0.6442000269889832},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5266000032424927},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.45509999990463257},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.4318000078201294},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4180999994277954},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.38929998874664307},{"id":"https://openalex.org/C2781316041","wikidata":"https://www.wikidata.org/wiki/Q1230584","display_name":"Diversity (politics)","level":2,"score":0.36500000953674316},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.34599998593330383},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.3431999981403351},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3352000117301941},{"id":"https://openalex.org/C188147891","wikidata":"https://www.wikidata.org/wiki/Q147638","display_name":"Cognitive science","level":1,"score":0.2930999994277954},{"id":"https://openalex.org/C2781089630","wikidata":"https://www.wikidata.org/wiki/Q21856745","display_name":"Realization (probability)","level":2,"score":0.2809999883174896},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.2540000081062317}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.28351","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.28351","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.28351","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.28351","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Reduced inequalities","id":"https://metadata.un.org/sdg/10","score":0.7973852157592773}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Subjective":[0],"judgments":[1],"are":[2,33,77,107],"part":[3],"of":[4,20,89,97,105,128],"several":[5],"NLP":[6,71],"datasets":[7],"and":[8,73,100,114,125],"recent":[9],"work":[10],"is":[11,68,119],"increasingly":[12],"prioritizing":[13],"models":[14],"whose":[15],"outputs":[16],"reflect":[17],"this":[18],"diversity":[19],"perspectives.":[21,40],"Such":[22],"responses":[23],"allow":[24],"us":[25],"to":[26,122],"shed":[27],"light":[28],"on":[29],"minority":[30],"voices,":[31],"which":[32],"frequently":[34],"marginalized":[35],"or":[36],"obscured":[37],"by":[38],"dominant":[39],"It":[41],"remains":[42],"a":[43,80,126],"question":[44],"whether":[45,117],"our":[46],"evaluation":[47,59],"practices":[48],"align":[49],"with":[50],"these":[51],"models'":[52],"objectives.":[53],"This":[54],"position":[55],"paper":[56],"proposes":[57],"seven":[58],"desiderata":[60,76],"for":[61],"subjectivity-sensitive":[62],"models,":[63],"rooted":[64],"in":[65,70,79,84],"how":[66],"subjectivity":[67,106,118],"represented":[69],"data":[72],"models.":[74,91],"The":[75],"constructed":[78],"top-down":[81],"approach,":[82],"keeping":[83],"mind":[85],"the":[86,94,110,123],"user-centric":[87],"impact":[88],"such":[90],"We":[92],"scan":[93],"experimental":[95],"setup":[96],"60":[98],"papers":[99],"show":[101],"that":[102],"various":[103],"aspects":[104],"still":[108],"understudied:":[109],"distinction":[111],"between":[112,130],"ambiguous":[113],"polyphonic":[115],"input,":[116],"effectively":[120],"expressed":[121],"user,":[124],"lack":[127],"interplay":[129],"different":[131],"desiderata,":[132],"amongst":[133],"other":[134],"gaps.":[135]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-04-02T00:00:00"}