{"id":"https://openalex.org/W4385260961","doi":"https://doi.org/10.1145/3600211.3604655","title":"How do you feel? Measuring User-Perceived Value for Rejecting Machine Decisions in Hate Speech Detection","display_name":"How do you feel? Measuring User-Perceived Value for Rejecting Machine Decisions in Hate Speech Detection","publication_year":2023,"publication_date":"2023-08-08","ids":{"openalex":"https://openalex.org/W4385260961","doi":"https://doi.org/10.1145/3600211.3604655"},"language":"en","primary_location":{"id":"doi:10.1145/3600211.3604655","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3600211.3604655","pdf_url":null,"source":null,"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 AAAI/ACM Conference on AI, Ethics, and Society","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3600211.3604655","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5051994016","display_name":"Philippe Lammerts","orcid":"https://orcid.org/0000-0003-3164-5592"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":true,"raw_author_name":"Philippe Lammerts","raw_affiliation_strings":["Delft University of Technology, Netherlands"],"affiliations":[{"raw_affiliation_string":"Delft University of Technology, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081679057","display_name":"Philip Lippmann","orcid":"https://orcid.org/0000-0002-0139-1061"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Philip Lippmann","raw_affiliation_strings":["Delft University of Technology, Netherlands"],"affiliations":[{"raw_affiliation_string":"Delft University of Technology, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008545327","display_name":"Yen-Chia Hsu","orcid":"https://orcid.org/0000-0002-8016-7534"},"institutions":[{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Yen-Chia Hsu","raw_affiliation_strings":["University of Amsterdam, Netherlands"],"affiliations":[{"raw_affiliation_string":"University of Amsterdam, Netherlands","institution_ids":["https://openalex.org/I887064364"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086585292","display_name":"Fabio Casati","orcid":"https://orcid.org/0000-0001-7591-9562"},"institutions":[{"id":"https://openalex.org/I4210157325","display_name":"HealthNow New York","ror":"https://ror.org/05wmmcx08","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I4210157325"]},{"id":"https://openalex.org/I4210167039","display_name":"ServiceNow (United States)","ror":"https://ror.org/05xr0bc04","country_code":"US","type":"company","lineage":["https://openalex.org/I4210167039"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Fabio Casati","raw_affiliation_strings":["ServiceNow, USA"],"affiliations":[{"raw_affiliation_string":"ServiceNow, USA","institution_ids":["https://openalex.org/I4210157325","https://openalex.org/I4210167039"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5029123550","display_name":"Jie Yang","orcid":"https://orcid.org/0000-0002-0350-0313"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jie Yang","raw_affiliation_strings":["Delft University of Technology, USA"],"affiliations":[{"raw_affiliation_string":"Delft University of Technology, USA","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5051994016"],"corresponding_institution_ids":["https://openalex.org/I98358874"],"apc_list":null,"apc_paid":null,"fwci":0.3516,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":{"value":0.65072088,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"834","last_page":"844"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12262","display_name":"Hate Speech and Cyberbullying Detection","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12262","display_name":"Hate Speech and Cyberbullying Detection","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9829000234603882,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7771086692810059},{"id":"https://openalex.org/keywords/moderation","display_name":"Moderation","score":0.6641985774040222},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.6506168842315674},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6016516089439392},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.6005796194076538},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5975927114486694},{"id":"https://openalex.org/keywords/human\u2013machine-system","display_name":"Human\u2013machine system","score":0.5542871952056885},{"id":"https://openalex.org/keywords/perception","display_name":"Perception","score":0.5393806099891663},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.47763362526893616},{"id":"https://openalex.org/keywords/value","display_name":"Value (mathematics)","score":0.4744473695755005},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.32539916038513184},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.09197956323623657},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.0866180956363678}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7771086692810059},{"id":"https://openalex.org/C93225998","wikidata":"https://www.wikidata.org/wiki/Q1941972","display_name":"Moderation","level":2,"score":0.6641985774040222},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.6506168842315674},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6016516089439392},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.6005796194076538},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5975927114486694},{"id":"https://openalex.org/C146047270","wikidata":"https://www.wikidata.org/wiki/Q469666","display_name":"Human\u2013machine system","level":2,"score":0.5542871952056885},{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.5393806099891663},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.47763362526893616},{"id":"https://openalex.org/C2776291640","wikidata":"https://www.wikidata.org/wiki/Q2912517","display_name":"Value (mathematics)","level":2,"score":0.4744473695755005},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.32539916038513184},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.09197956323623657},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0866180956363678},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0},{"id":"https://openalex.org/C169760540","wikidata":"https://www.wikidata.org/wiki/Q207011","display_name":"Neuroscience","level":1,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":6,"locations":[{"id":"doi:10.1145/3600211.3604655","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3600211.3604655","pdf_url":null,"source":null,"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 AAAI/ACM Conference on AI, Ethics, and Society","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2307.11806","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2307.11806","pdf_url":"https://arxiv.org/pdf/2307.11806","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:dare.uva.nl:openaire/8f8cebd2-44df-471e-abb6-72203b656c03","is_oa":true,"landing_page_url":"https://handle.uba.uva.nl/personal/pure/en/publications/how-do-you-feel-measuring-userperceived-value-for-rejecting-machine-decisions-in-hate-speech-detection(8f8cebd2-44df-471e-abb6-72203b656c03).html","pdf_url":"https://pure.uva.nl/ws/files/178792451/3600211.3604655.pdf","source":{"id":"https://openalex.org/S4306400088","display_name":"UvA-DARE (University of Amsterdam)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I887064364","host_organization_name":"University of Amsterdam","host_organization_lineage":["https://openalex.org/I887064364"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Lammerts, P, Lippmann, P, Hsu, Y-C, Casati, F & Yang, J 2023, How do you feel? Measuring User-Perceived Value for Rejecting Machine Decisions in Hate Speech Detection. in AIES '23 : proceedings of the 2023 AAAI/ACM Conference on AI, Ethics, and Society : August 8-10, 2023, Montreal, Canada. Association for Computing Machinery, New York, New York , pp. 834-844, 2023 AAAI / ACM Conference on Artificial Intelligence, Ethics, and Society, AIES 2023, Montreal, Canada, 8/08/23. https://doi.org/10.1145/3600211.3604655","raw_type":"info:eu-repo/semantics/publishedVersion"},{"id":"pmh:oai:dare.uva.nl:publications/8f8cebd2-44df-471e-abb6-72203b656c03","is_oa":false,"landing_page_url":"https://hdl.handle.net/11245.1/8f8cebd2-44df-471e-abb6-72203b656c03","pdf_url":null,"source":{"id":"https://openalex.org/S4306400088","display_name":"UvA-DARE (University of Amsterdam)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I887064364","host_organization_name":"University of Amsterdam","host_organization_lineage":["https://openalex.org/I887064364"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Lammerts, P, Lippmann, P, Hsu, Y-C, Casati, F & Yang, J 2023, How do you feel? Measuring User-Perceived Value for Rejecting Machine Decisions in Hate Speech Detection. in AIES '23 : proceedings of the 2023 AAAI/ACM Conference on AI, Ethics, and Society : August 8-10, 2023, Montreal, Canada. Association for Computing Machinery, New York, New York , pp. 834-844, 2023 AAAI / ACM Conference on Artificial Intelligence, Ethics, and Society, AIES 2023, Montreal, Canada, 8/08/23. https://doi.org/10.1145/3600211.3604655","raw_type":"info:eu-repo/semantics/publishedVersion"},{"id":"pmh:oai:iris.unitn.it:11572/397739","is_oa":true,"landing_page_url":"https://dl.acm.org/doi/10.1145/3600211.3604655","pdf_url":null,"source":{"id":"https://openalex.org/S4306401913","display_name":"Institutional Research Information System (Universit\u00e0 degli Studi di Trento)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I193223587","host_organization_name":"University of Trento","host_organization_lineage":["https://openalex.org/I193223587"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/conferenceObject"},{"id":"pmh:oai:tudelft.nl:uuid:5fc082f5-0993-4136-a6f9-576294c6b275","is_oa":true,"landing_page_url":"http://resolver.tudelft.nl/uuid:5fc082f5-0993-4136-a6f9-576294c6b275","pdf_url":null,"source":{"id":"https://openalex.org/S4306400906","display_name":"Research Repository (Delft University of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I98358874","host_organization_name":"Delft University of Technology","host_organization_lineage":["https://openalex.org/I98358874"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"conference paper"}],"best_oa_location":{"id":"doi:10.1145/3600211.3604655","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3600211.3604655","pdf_url":null,"source":null,"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 AAAI/ACM Conference on AI, Ethics, and Society","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","score":0.7599999904632568,"id":"https://metadata.un.org/sdg/16"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":45,"referenced_works":["https://openalex.org/W206216833","https://openalex.org/W1679804803","https://openalex.org/W1938907317","https://openalex.org/W1966547126","https://openalex.org/W1983399107","https://openalex.org/W1991158387","https://openalex.org/W2010156196","https://openalex.org/W2012378416","https://openalex.org/W2036706085","https://openalex.org/W2074624540","https://openalex.org/W2094485533","https://openalex.org/W2118739093","https://openalex.org/W2137560901","https://openalex.org/W2282821441","https://openalex.org/W2473555522","https://openalex.org/W2569307494","https://openalex.org/W2606766508","https://openalex.org/W2734639922","https://openalex.org/W2792183666","https://openalex.org/W2798782484","https://openalex.org/W2887782043","https://openalex.org/W2889016530","https://openalex.org/W2898829623","https://openalex.org/W2898911770","https://openalex.org/W2923345718","https://openalex.org/W2943327675","https://openalex.org/W2946076379","https://openalex.org/W2954226438","https://openalex.org/W2954479967","https://openalex.org/W2961730454","https://openalex.org/W2962797668","https://openalex.org/W2981695923","https://openalex.org/W2998598262","https://openalex.org/W2999331643","https://openalex.org/W3034854469","https://openalex.org/W3108753913","https://openalex.org/W3122907670","https://openalex.org/W3128442340","https://openalex.org/W3157672405","https://openalex.org/W3210712343","https://openalex.org/W4283217613","https://openalex.org/W4300978593","https://openalex.org/W4306791975","https://openalex.org/W4385403451","https://openalex.org/W6637564071"],"related_works":["https://openalex.org/W2740268725","https://openalex.org/W3093710306","https://openalex.org/W750217911","https://openalex.org/W4206949499","https://openalex.org/W2296969234","https://openalex.org/W4226254121","https://openalex.org/W4388229525","https://openalex.org/W131533274","https://openalex.org/W4313163826","https://openalex.org/W2478524818"],"abstract_inverted_index":{"Hate":[0],"speech":[1,110],"moderation":[2,77],"remains":[3],"a":[4,66,88,121,150,190],"challenging":[5],"task":[6,38],"for":[7,37,75,135],"social":[8],"media":[9],"platforms.":[10],"Human-AI":[11],"collaborative":[12],"systems":[13],"offer":[14],"the":[15,19,25,46,106,117,132,186,203],"potential":[16],"to":[17,30,56,95,178,184,202],"combine":[18],"strengths":[20],"of":[21,27,48,99,108,153,156,208],"humans\u2019":[22],"reliability":[23],"and":[24,101,194],"scalability":[26],"machine":[28,73,84,103,140,157,182],"learning":[29],"tackle":[31],"this":[32,62],"issue":[33],"effectively.":[34],"While":[35],"methods":[36],"handover":[39],"in":[40,105,174],"human-AI":[41,164],"collaboration":[42],"exist":[43],"that":[44,70,145,169],"consider":[45],"costs":[47],"incorrect":[49,102],"predictions,":[50],"insufficient":[51],"attention":[52],"has":[53],"been":[54],"paid":[55],"accurately":[57],"estimating":[58],"these":[59],"costs.":[60],"In":[61],"work,":[63],"we":[64,124,166],"propose":[65],"value-sensitive":[67],"rejection":[68],"mechanism":[69],"automatically":[71],"rejects":[72,119],"decisions":[74,104,183],"human":[76],"based":[78],"on":[79],"users\u2019":[80],"value":[81,162,189],"perceptions":[82],"regarding":[83],"decisions.":[85,141,158],"We":[86],"conduct":[87],"crowdsourced":[89],"survey":[90],"study":[91],"with":[92,139],"160":[93],"participants":[94],"evaluate":[96],"their":[97],"perception":[98,155],"correct":[100],"domain":[107],"hate":[109],"detection,":[111],"as":[112,114,131,200],"well":[113],"occurrences":[115],"where":[116],"system":[118],"making":[120],"prediction.":[122],"Here,":[123],"introduce":[125],"Magnitude":[126,146],"Estimation,":[127],"an":[128],"unbounded":[129],"scale,":[130],"preferred":[133],"method":[134],"measuring":[136],"user":[137],"(dis)agreement":[138],"Our":[142],"results":[143],"show":[144,168],"Estimation":[147],"can":[148,171,192],"provide":[149],"reliable":[151],"measurement":[152],"participants\u2019":[154],"By":[159],"integrating":[160],"user-perceived":[161],"into":[163],"collaboration,":[165],"further":[167],"it":[170],"guide":[172],"us":[173],"1)":[175],"determining":[176],"when":[177],"accept":[179],"or":[180],"reject":[181],"obtain":[185],"optimal":[187],"total":[188],"model":[191,209],"deliver":[193],"2)":[195],"selecting":[196],"better":[197],"classification":[198],"models":[199],"compared":[201],"more":[204],"widely":[205],"used":[206],"target":[207],"accuracy.":[210]},"counts_by_year":[{"year":2023,"cited_by_count":2}],"updated_date":"2026-03-20T23:20:44.827607","created_date":"2025-10-10T00:00:00"}