{"id":"https://openalex.org/W4390957764","doi":"https://doi.org/10.1109/besc59560.2023.10386355","title":"Auditing Machine Behaviors: Does a diverse crowd result in different social norms?","display_name":"Auditing Machine Behaviors: Does a diverse crowd result in different social norms?","publication_year":2023,"publication_date":"2023-10-30","ids":{"openalex":"https://openalex.org/W4390957764","doi":"https://doi.org/10.1109/besc59560.2023.10386355"},"language":"en","primary_location":{"id":"doi:10.1109/besc59560.2023.10386355","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/besc59560.2023.10386355","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 10th International Conference on Behavioural and Social Computing (BESC)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.1109/BESC59560.2023.10386355","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5068282303","display_name":"Evgenia Christoforou","orcid":"https://orcid.org/0000-0003-0455-9357"},"institutions":[{"id":"https://openalex.org/I4210105896","display_name":"Cyprus Research and Innovation Center (Cyprus)","ror":"https://ror.org/01ha10g31","country_code":"CY","type":"company","lineage":["https://openalex.org/I4210105896"]}],"countries":["CY"],"is_corresponding":true,"raw_author_name":"Evgenia Christoforou","raw_affiliation_strings":["CYENS - Centre of Excellence,Nicosia,Cyprus","CYENS - Centre of Excellence, Nicosia, Cyprus"],"affiliations":[{"raw_affiliation_string":"CYENS - Centre of Excellence,Nicosia,Cyprus","institution_ids":["https://openalex.org/I4210105896"]},{"raw_affiliation_string":"CYENS - Centre of Excellence, Nicosia, Cyprus","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025866969","display_name":"Kalia Orphanou","orcid":"https://orcid.org/0000-0003-1351-2489"},"institutions":[{"id":"https://openalex.org/I148518341","display_name":"Cyprus Institute","ror":"https://ror.org/01q8k8p90","country_code":"CY","type":"other","lineage":["https://openalex.org/I148518341"]},{"id":"https://openalex.org/I59746437","display_name":"Open University of Cyprus","ror":"https://ror.org/033sm2k57","country_code":"CY","type":"education","lineage":["https://openalex.org/I59746437"]}],"countries":["CY"],"is_corresponding":false,"raw_author_name":"Kalia Orphanou","raw_affiliation_strings":["Open University of Cyprus,The Cyprus Institute of Neurology & Genetics,Nicosia,Cyprus"],"affiliations":[{"raw_affiliation_string":"Open University of Cyprus,The Cyprus Institute of Neurology & Genetics,Nicosia,Cyprus","institution_ids":["https://openalex.org/I59746437","https://openalex.org/I148518341"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020913819","display_name":"Nicolas Nicolaou","orcid":"https://orcid.org/0000-0001-7540-784X"},"institutions":[{"id":"https://openalex.org/I17389662","display_name":"University of Nicosia","ror":"https://ror.org/04v18t651","country_code":"CY","type":"education","lineage":["https://openalex.org/I17389662"]}],"countries":["CY"],"is_corresponding":false,"raw_author_name":"Nicolas Nicolaou","raw_affiliation_strings":["Algolysis Ltd,Nicosia,Cyprus","Algolysis Ltd, Nicosia, Cyprus"],"affiliations":[{"raw_affiliation_string":"Algolysis Ltd,Nicosia,Cyprus","institution_ids":["https://openalex.org/I17389662"]},{"raw_affiliation_string":"Algolysis Ltd, Nicosia, Cyprus","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015479492","display_name":"Efstathios Stavrakis","orcid":"https://orcid.org/0000-0002-9213-7690"},"institutions":[{"id":"https://openalex.org/I17389662","display_name":"University of Nicosia","ror":"https://ror.org/04v18t651","country_code":"CY","type":"education","lineage":["https://openalex.org/I17389662"]}],"countries":["CY"],"is_corresponding":false,"raw_author_name":"Efstathios Stavrakis","raw_affiliation_strings":["Algolysis Ltd,Nicosia,Cyprus","Algolysis Ltd, Nicosia, Cyprus"],"affiliations":[{"raw_affiliation_string":"Algolysis Ltd,Nicosia,Cyprus","institution_ids":["https://openalex.org/I17389662"]},{"raw_affiliation_string":"Algolysis Ltd, Nicosia, Cyprus","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5028278233","display_name":"Jahna Otterbacher","orcid":"https://orcid.org/0000-0002-7655-7118"},"institutions":[{"id":"https://openalex.org/I59746437","display_name":"Open University of Cyprus","ror":"https://ror.org/033sm2k57","country_code":"CY","type":"education","lineage":["https://openalex.org/I59746437"]}],"countries":["CY"],"is_corresponding":false,"raw_author_name":"Jahna Otterbacher","raw_affiliation_strings":["Open University of Cyprus,CYENS - Centre of Excellence,Nicosia,Cyprus","CYENS - Centre of Excellence, Open University of Cyprus, Nicosia, Cyprus"],"affiliations":[{"raw_affiliation_string":"Open University of Cyprus,CYENS - Centre of Excellence,Nicosia,Cyprus","institution_ids":["https://openalex.org/I59746437"]},{"raw_affiliation_string":"CYENS - Centre of Excellence, Open University of Cyprus, Nicosia, Cyprus","institution_ids":["https://openalex.org/I59746437"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5068282303"],"corresponding_institution_ids":["https://openalex.org/I4210105896"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.21767792,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"22","issue":null,"first_page":"1","last_page":"8"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9983999729156494,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9983999729156494,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10646","display_name":"Experimental Behavioral Economics Studies","score":0.9907000064849854,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7877458333969116},{"id":"https://openalex.org/keywords/audit","display_name":"Audit","score":0.7737988233566284},{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.6918812990188599},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.6723507642745972},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.6195350885391235},{"id":"https://openalex.org/keywords/perspective","display_name":"Perspective (graphical)","score":0.570534884929657},{"id":"https://openalex.org/keywords/black-box","display_name":"Black box","score":0.5566085577011108},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.4318881332874298},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.4106692671775818},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.336900532245636},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3120157718658447},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.2801668643951416},{"id":"https://openalex.org/keywords/business","display_name":"Business","score":0.10108518600463867}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7877458333969116},{"id":"https://openalex.org/C199521495","wikidata":"https://www.wikidata.org/wiki/Q181487","display_name":"Audit","level":2,"score":0.7737988233566284},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.6918812990188599},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.6723507642745972},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.6195350885391235},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.570534884929657},{"id":"https://openalex.org/C94966114","wikidata":"https://www.wikidata.org/wiki/Q29256","display_name":"Black box","level":2,"score":0.5566085577011108},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4318881332874298},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.4106692671775818},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.336900532245636},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3120157718658447},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.2801668643951416},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.10108518600463867},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C121955636","wikidata":"https://www.wikidata.org/wiki/Q4116214","display_name":"Accounting","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/besc59560.2023.10386355","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/besc59560.2023.10386355","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 10th International Conference on Behavioural and Social Computing (BESC)","raw_type":"proceedings-article"},{"id":"pmh:oai:zenodo.org:13235630","is_oa":true,"landing_page_url":"https://doi.org/10.1109/BESC59560.2023.10386355","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"BESC, 10th International Conference on Behavioural and Social Computing, Larnaca, Cyprus","raw_type":"info:eu-repo/semantics/conferencePaper"}],"best_oa_location":{"id":"pmh:oai:zenodo.org:13235630","is_oa":true,"landing_page_url":"https://doi.org/10.1109/BESC59560.2023.10386355","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"BESC, 10th International Conference on Behavioural and Social Computing, Larnaca, Cyprus","raw_type":"info:eu-repo/semantics/conferencePaper"},"sustainable_development_goals":[{"score":0.5,"display_name":"Gender equality","id":"https://metadata.un.org/sdg/5"}],"awards":[],"funders":[{"id":"https://openalex.org/F4320330084","display_name":"Research and Innovation Foundation","ror":null}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":23,"referenced_works":["https://openalex.org/W1964981411","https://openalex.org/W2014352947","https://openalex.org/W2149252982","https://openalex.org/W2256229999","https://openalex.org/W2799258637","https://openalex.org/W2885858722","https://openalex.org/W2915177913","https://openalex.org/W2942048673","https://openalex.org/W2964316623","https://openalex.org/W2966344724","https://openalex.org/W3094217062","https://openalex.org/W3159435787","https://openalex.org/W3160735296","https://openalex.org/W3198804996","https://openalex.org/W3204393347","https://openalex.org/W3206420877","https://openalex.org/W4212884277","https://openalex.org/W4285225692","https://openalex.org/W4288359825","https://openalex.org/W4309618902","https://openalex.org/W4317942025","https://openalex.org/W4321456513","https://openalex.org/W6849434732"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W135177976","https://openalex.org/W4384486036","https://openalex.org/W1503094549","https://openalex.org/W2337920774","https://openalex.org/W4286908577","https://openalex.org/W2886410948","https://openalex.org/W2025875869","https://openalex.org/W4318823662","https://openalex.org/W3207526114"],"abstract_inverted_index":{"Many":[0],"intelligent":[1],"systems":[2],"depend":[3],"on":[4],"black-box":[5,65,149],"services":[6,19],"for":[7,20,48,128,137,145],"achieving":[8],"goals.":[9],"For":[10,105],"example,":[11,106],"Decision":[12],"Support":[13],"Systems":[14],"(DSS)":[15],"incorporate":[16],"image":[17,26,66,78,150],"tagging":[18],"translating":[21],"the":[22,32,35,44,61,64,83,99,107,130,161,203,210,214,217,226],"content":[23],"of":[24,34,46,63,82,88,102,132,198,216],"an":[25,77,135],"into":[27],"useful":[28],"information":[29],"that":[30,73,208],"serves":[31],"purpose":[33],"system":[36],"under":[37],"development":[38],"(e.g.,":[39],"facilitating":[40],"hiring":[41,122,171],"decisions).":[42],"Auditing":[43],"results":[45],"DSS":[47,84,211],"fairness":[49,89,222],"can":[50,90,110],"thus":[51],"become":[52],"a":[53,103,115,121,126,153,170,175,220],"cumbersome":[54],"process":[55],"as":[56,134],"they":[57],"are":[58,80],"correlated":[59],"with":[60],"output":[62],"tagger.":[67],"In":[68],"this":[69],"work,":[70],"we":[71,156,181],"assume":[72],"tags":[74,146,165],"generated":[75,147],"by":[76,148,190],"tagger":[79],"part":[81],"and":[85,143,173,200,213,228],"in":[86,114,120,166],"terms":[87],"be":[91,111],"considered":[92,112],"socially":[93,141],"acceptable":[94,142],"or":[95],"unacceptable":[96,144],"conforming":[97],"to":[98,158,225],"varying":[100,209],"goals":[101],"DSS.":[104,177],"tag":[108],"\"good-looking\"":[109],"appropriate":[113],"dating":[116,176],"context":[117],"but":[118],"inappropriate":[119],"context.":[123],"We":[124],"propose":[125],"methodology":[127],"leveraging":[129],"use":[131],"crowdsourcing":[133],"approach":[136],"auditing":[138,204],"what":[139],"is":[140],"taggers.":[151],"As":[152],"first":[154],"step,":[155],"attempt":[157],"understand":[159],"how":[160,183],"crowdworkers":[162,191],"perceive":[163],"(un)acceptable":[164],"two":[167],"scenarios:":[168],"(i)":[169],"DSS,":[172],"(ii)":[174],"Through":[178],"our":[179,186],"analysis,":[180],"investigate":[182],"answers":[184],"(in":[185],"case":[187],"tags)":[188],"received":[189],"representing":[192],"different":[193,221],"demographic":[194],"groups":[195],"(i.e.,":[196],"country":[197],"residence":[199],"gender)":[201],"impact":[202],"process.":[205],"Results":[206],"indicate":[207],"scenario,":[212],"demographics":[215],"crowdworkers,":[218],"yields":[219],"perspective":[223],"concurrent":[224],"\"acceptable\"":[227],"\"unacceptable\"":[229],"social":[230],"norms.":[231]},"counts_by_year":[],"updated_date":"2026-03-20T23:20:44.827607","created_date":"2025-10-10T00:00:00"}