{"id":"https://openalex.org/W4399364346","doi":"https://doi.org/10.1145/3630106.3659018","title":"Balancing Act: Evaluating People\u2019s Perceptions of Fair Ranking Metrics","display_name":"Balancing Act: Evaluating People\u2019s Perceptions of Fair Ranking Metrics","publication_year":2024,"publication_date":"2024-06-03","ids":{"openalex":"https://openalex.org/W4399364346","doi":"https://doi.org/10.1145/3630106.3659018"},"language":"en","primary_location":{"id":"doi:10.1145/3630106.3659018","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3630106.3659018","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3630106.3659018","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"The 2024 ACM Conference on Fairness, Accountability, and Transparency","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3630106.3659018","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5089529025","display_name":"Mallak Alkhathlan","orcid":"https://orcid.org/0000-0003-1933-3502"},"institutions":[{"id":"https://openalex.org/I107077323","display_name":"Worcester Polytechnic Institute","ror":"https://ror.org/05ejpqr48","country_code":"US","type":"education","lineage":["https://openalex.org/I107077323"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Mallak Alkhathlan","raw_affiliation_strings":["Worcester Polytechnic Institute, United States"],"affiliations":[{"raw_affiliation_string":"Worcester Polytechnic Institute, United States","institution_ids":["https://openalex.org/I107077323"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041530401","display_name":"Kathleen Cachel","orcid":"https://orcid.org/0000-0002-1260-0914"},"institutions":[{"id":"https://openalex.org/I107077323","display_name":"Worcester Polytechnic Institute","ror":"https://ror.org/05ejpqr48","country_code":"US","type":"education","lineage":["https://openalex.org/I107077323"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kathleen Cachel","raw_affiliation_strings":["Worcester Polytechnic Institute, USA"],"affiliations":[{"raw_affiliation_string":"Worcester Polytechnic Institute, USA","institution_ids":["https://openalex.org/I107077323"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066774534","display_name":"Hilson Shrestha","orcid":"https://orcid.org/0009-0006-6603-6606"},"institutions":[{"id":"https://openalex.org/I107077323","display_name":"Worcester Polytechnic Institute","ror":"https://ror.org/05ejpqr48","country_code":"US","type":"education","lineage":["https://openalex.org/I107077323"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hilson Shrestha","raw_affiliation_strings":["Worcester Polytechnic Institute, USA"],"affiliations":[{"raw_affiliation_string":"Worcester Polytechnic Institute, USA","institution_ids":["https://openalex.org/I107077323"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087146581","display_name":"Lane Harrison","orcid":"https://orcid.org/0000-0003-3029-2799"},"institutions":[{"id":"https://openalex.org/I107077323","display_name":"Worcester Polytechnic Institute","ror":"https://ror.org/05ejpqr48","country_code":"US","type":"education","lineage":["https://openalex.org/I107077323"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Lane Harrison","raw_affiliation_strings":["Worcester Polytechnic Institute, USA"],"affiliations":[{"raw_affiliation_string":"Worcester Polytechnic Institute, USA","institution_ids":["https://openalex.org/I107077323"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5008269094","display_name":"Elke A. Rundensteiner","orcid":"https://orcid.org/0000-0001-5375-9254"},"institutions":[{"id":"https://openalex.org/I107077323","display_name":"Worcester Polytechnic Institute","ror":"https://ror.org/05ejpqr48","country_code":"US","type":"education","lineage":["https://openalex.org/I107077323"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Elke Rundensteiner","raw_affiliation_strings":["Worcester Polytechnic Institute, USA"],"affiliations":[{"raw_affiliation_string":"Worcester Polytechnic Institute, USA","institution_ids":["https://openalex.org/I107077323"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5089529025"],"corresponding_institution_ids":["https://openalex.org/I107077323"],"apc_list":null,"apc_paid":null,"fwci":3.0463,"has_fulltext":false,"cited_by_count":6,"citation_normalized_percentile":{"value":0.92139786,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"1940","last_page":"1970"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10646","display_name":"Experimental Behavioral Economics Studies","score":0.9692000150680542,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11182","display_name":"Auction Theory and Applications","score":0.944599986076355,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/operationalization","display_name":"Operationalization","score":0.8295340538024902},{"id":"https://openalex.org/keywords/meritocracy","display_name":"Meritocracy","score":0.8224010467529297},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.7193272113800049},{"id":"https://openalex.org/keywords/preference","display_name":"Preference","score":0.6515883207321167},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.624384343624115},{"id":"https://openalex.org/keywords/perception","display_name":"Perception","score":0.5725656747817993},{"id":"https://openalex.org/keywords/scholarship","display_name":"Scholarship","score":0.5426976084709167},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.48087266087532043},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.4193205237388611},{"id":"https://openalex.org/keywords/social-psychology","display_name":"Social psychology","score":0.4183327853679657},{"id":"https://openalex.org/keywords/economics","display_name":"Economics","score":0.2184053659439087},{"id":"https://openalex.org/keywords/microeconomics","display_name":"Microeconomics","score":0.18345114588737488},{"id":"https://openalex.org/keywords/marketing","display_name":"Marketing","score":0.1689380407333374},{"id":"https://openalex.org/keywords/business","display_name":"Business","score":0.15818795561790466},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.14980322122573853}],"concepts":[{"id":"https://openalex.org/C9354725","wikidata":"https://www.wikidata.org/wiki/Q286017","display_name":"Operationalization","level":2,"score":0.8295340538024902},{"id":"https://openalex.org/C38377331","wikidata":"https://www.wikidata.org/wiki/Q178079","display_name":"Meritocracy","level":2,"score":0.8224010467529297},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.7193272113800049},{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.6515883207321167},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.624384343624115},{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.5725656747817993},{"id":"https://openalex.org/C2778061430","wikidata":"https://www.wikidata.org/wiki/Q188823","display_name":"Scholarship","level":2,"score":0.5426976084709167},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.48087266087532043},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.4193205237388611},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.4183327853679657},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.2184053659439087},{"id":"https://openalex.org/C175444787","wikidata":"https://www.wikidata.org/wiki/Q39072","display_name":"Microeconomics","level":1,"score":0.18345114588737488},{"id":"https://openalex.org/C162853370","wikidata":"https://www.wikidata.org/wiki/Q39809","display_name":"Marketing","level":1,"score":0.1689380407333374},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.15818795561790466},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.14980322122573853},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C169760540","wikidata":"https://www.wikidata.org/wiki/Q207011","display_name":"Neuroscience","level":1,"score":0.0},{"id":"https://openalex.org/C34447519","wikidata":"https://www.wikidata.org/wiki/Q179522","display_name":"Market economy","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C50522688","wikidata":"https://www.wikidata.org/wiki/Q189833","display_name":"Economic growth","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3630106.3659018","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3630106.3659018","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3630106.3659018","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"The 2024 ACM Conference on Fairness, Accountability, and Transparency","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3630106.3659018","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3630106.3659018","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3630106.3659018","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"The 2024 ACM Conference on Fairness, Accountability, and Transparency","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.699999988079071,"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4399364346.pdf"},"referenced_works_count":67,"referenced_works":["https://openalex.org/W576751684","https://openalex.org/W1517183858","https://openalex.org/W1593806347","https://openalex.org/W1601972889","https://openalex.org/W1648937643","https://openalex.org/W1990593398","https://openalex.org/W2026019770","https://openalex.org/W2100960835","https://openalex.org/W2146167636","https://openalex.org/W2149252982","https://openalex.org/W2544318541","https://openalex.org/W2704480242","https://openalex.org/W2705765409","https://openalex.org/W2753845591","https://openalex.org/W2787991113","https://openalex.org/W2809878087","https://openalex.org/W2888034815","https://openalex.org/W2914202940","https://openalex.org/W2947160886","https://openalex.org/W2950173087","https://openalex.org/W2963588812","https://openalex.org/W2964316623","https://openalex.org/W2983996708","https://openalex.org/W2992319600","https://openalex.org/W2995006168","https://openalex.org/W2997398218","https://openalex.org/W3005196550","https://openalex.org/W3013571594","https://openalex.org/W3029022080","https://openalex.org/W3035885149","https://openalex.org/W3047563234","https://openalex.org/W3086107884","https://openalex.org/W3102092462","https://openalex.org/W3102518922","https://openalex.org/W3103891807","https://openalex.org/W3126632784","https://openalex.org/W3133874049","https://openalex.org/W3135484384","https://openalex.org/W3136620301","https://openalex.org/W3138463903","https://openalex.org/W3152436735","https://openalex.org/W3163696583","https://openalex.org/W3171889065","https://openalex.org/W3174602246","https://openalex.org/W3187177811","https://openalex.org/W3189108721","https://openalex.org/W3202755633","https://openalex.org/W4213457393","https://openalex.org/W4225082379","https://openalex.org/W4281856887","https://openalex.org/W4283171263","https://openalex.org/W4284690424","https://openalex.org/W4288359825","https://openalex.org/W4289328212","https://openalex.org/W4289533941","https://openalex.org/W4308709498","https://openalex.org/W4309618220","https://openalex.org/W4320853676","https://openalex.org/W4366549927","https://openalex.org/W4366550208","https://openalex.org/W4367046988","https://openalex.org/W4376988619","https://openalex.org/W4380319549","https://openalex.org/W4380320234","https://openalex.org/W4380369890","https://openalex.org/W4386955434","https://openalex.org/W7042961576"],"related_works":["https://openalex.org/W1990443116","https://openalex.org/W4299139262","https://openalex.org/W3047348699","https://openalex.org/W2039692704","https://openalex.org/W4255731772","https://openalex.org/W4293754902","https://openalex.org/W2931602588","https://openalex.org/W51364034","https://openalex.org/W2898073868","https://openalex.org/W4284663758"],"abstract_inverted_index":{"Algorithmic":[0],"decision-making":[1,155],"using":[2],"rankings\u2014":[3],"prevalent":[4],"in":[5,115,131,154],"areas":[6],"from":[7],"hiring":[8],"and":[9,31,98,143],"bail":[10],"to":[11,106],"university":[12],"admissions\u2014":[13],"raises":[14],"concerns":[15,100],"of":[16,29,51,72,146],"potential":[17],"bias.":[18],"In":[19,39,123],"this":[20],"paper,":[21],"we":[22,125],"explore":[23],"the":[24,48,81],"alignment":[25],"between":[26,137],"people\u2019s":[27,73],"perceptions":[28],"fairness":[30,34,50,77,102,130],"two":[32],"popular":[33],"metrics":[35,103],"designed":[36],"for":[37,75,120],"rankings.":[38,122],"a":[40,52,59,88,134],"crowdsourced":[41],"experiment":[42],"with":[43],"480":[44],"participants,":[45],"people":[46],"rated":[47],"perceived":[49],"hypothetical":[53],"scholarship":[54],"distribution":[55],"scenario.":[56],"Results":[57],"suggest":[58],"strong":[60],"inclination":[61],"towards":[62,140],"relying":[63],"on":[64],"explicit":[65],"score":[66],"values.":[67],"There":[68],"is":[69,133],"also":[70],"evidence":[71],"preference":[74],"one":[76],"metric,":[78,83],"NDKL,":[79],"over":[80],"other":[82,110],"ARP.":[84],"Qualitative":[85],"results":[86],"paint":[87],"more":[89],"complex":[90],"picture:":[91],"some":[92],"participants":[93,111],"endorse":[94],"meritocratic":[95],"award":[96],"schemes":[97],"express":[99],"about":[101],"being":[104],"used":[105],"modify":[107],"rankings;":[108],"while":[109],"acknowledge":[112],"socio-economic":[113],"factors":[114],"score-based":[116],"rankings":[117],"as":[118,152],"justification":[119],"adjusting":[121],"summary,":[124],"find":[126],"that":[127],"operationalizing":[128],"algorithmic":[129],"practice":[132],"balancing":[135],"act":[136],"mitigating":[138],"harms":[139],"marginalized":[141],"groups":[142],"societal":[144],"conventions":[145],"leveraging":[147],"traditional":[148],"performance":[149],"scores":[150],"such":[151],"grades":[153],"contexts.":[156]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":5}],"updated_date":"2026-04-14T08:04:32.555800","created_date":"2025-10-10T00:00:00"}