{"id":"https://openalex.org/W3157464625","doi":"https://doi.org/10.1145/3449149","title":"Prior and Prejudice","display_name":"Prior and Prejudice","publication_year":2021,"publication_date":"2021-04-13","ids":{"openalex":"https://openalex.org/W3157464625","doi":"https://doi.org/10.1145/3449149","mag":"3157464625"},"language":"en","primary_location":{"id":"doi:10.1145/3449149","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3449149","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3449149","source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3449149","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5047633303","display_name":"Ivan Stelmakh","orcid":"https://orcid.org/0000-0002-9237-7379"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ivan Stelmakh","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034272491","display_name":"Nihar B. Shah","orcid":"https://orcid.org/0000-0001-5158-9677"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nihar B. Shah","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100758181","display_name":"Aarti Singh","orcid":"https://orcid.org/0000-0002-8278-3672"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Aarti Singh","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5019928111","display_name":"Hal Daum\u00e9","orcid":"https://orcid.org/0000-0002-3760-345X"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hal Daum\u00e9","raw_affiliation_strings":["University of Maryland & Microsoft Research, College Park, MD, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Maryland & Microsoft Research, College Park, MD, USA","institution_ids":["https://openalex.org/I66946132","https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":3.1828,"has_fulltext":true,"cited_by_count":17,"citation_normalized_percentile":{"value":0.92026578,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":98},"biblio":{"volume":"5","issue":"CSCW1","first_page":"1","last_page":"17"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.9537000060081482,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.9537000060081482,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9501000046730042,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9435999989509583,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/clarity","display_name":"CLARITY","score":0.7353847026824951},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5449196100234985},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.48629364371299744},{"id":"https://openalex.org/keywords/originality","display_name":"Originality","score":0.47271984815597534},{"id":"https://openalex.org/keywords/population","display_name":"Population","score":0.46582624316215515},{"id":"https://openalex.org/keywords/skepticism","display_name":"Skepticism","score":0.46245312690734863},{"id":"https://openalex.org/keywords/peer-review","display_name":"Peer review","score":0.45248913764953613},{"id":"https://openalex.org/keywords/medical-education","display_name":"Medical education","score":0.4486989676952362},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.3444221615791321},{"id":"https://openalex.org/keywords/social-psychology","display_name":"Social psychology","score":0.25790321826934814},{"id":"https://openalex.org/keywords/medicine","display_name":"Medicine","score":0.21369990706443787},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.14535605907440186}],"concepts":[{"id":"https://openalex.org/C2777146004","wikidata":"https://www.wikidata.org/wiki/Q14949826","display_name":"CLARITY","level":2,"score":0.7353847026824951},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5449196100234985},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.48629364371299744},{"id":"https://openalex.org/C2776950860","wikidata":"https://www.wikidata.org/wiki/Q2914681","display_name":"Originality","level":3,"score":0.47271984815597534},{"id":"https://openalex.org/C2908647359","wikidata":"https://www.wikidata.org/wiki/Q2625603","display_name":"Population","level":2,"score":0.46582624316215515},{"id":"https://openalex.org/C18296254","wikidata":"https://www.wikidata.org/wiki/Q1395219","display_name":"Skepticism","level":2,"score":0.46245312690734863},{"id":"https://openalex.org/C138368954","wikidata":"https://www.wikidata.org/wiki/Q215028","display_name":"Peer review","level":2,"score":0.45248913764953613},{"id":"https://openalex.org/C509550671","wikidata":"https://www.wikidata.org/wiki/Q126945","display_name":"Medical education","level":1,"score":0.4486989676952362},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3444221615791321},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.25790321826934814},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.21369990706443787},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.14535605907440186},{"id":"https://openalex.org/C11012388","wikidata":"https://www.wikidata.org/wiki/Q170658","display_name":"Creativity","level":2,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C99454951","wikidata":"https://www.wikidata.org/wiki/Q932068","display_name":"Environmental health","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3449149","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3449149","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3449149","source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1145/3449149","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3449149","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3449149","source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.8199999928474426,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[{"id":"https://openalex.org/G2492749369","display_name":"CIF: Medium: Foundations of Learning from Paired Comparisons and Direct Queries","funder_award_id":"1763734","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G2835527574","display_name":null,"funder_award_id":"1942124","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G5895635206","display_name":null,"funder_award_id":"CIF: 1942124, CIF: 1763734","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G6671297155","display_name":null,"funder_award_id":"CAREER","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G7967194905","display_name":null,"funder_award_id":"CIF: 1763734","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320310207","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3157464625.pdf","grobid_xml":"https://content.openalex.org/works/W3157464625.grobid-xml"},"referenced_works_count":70,"referenced_works":["https://openalex.org/W5788460","https://openalex.org/W158727920","https://openalex.org/W174896270","https://openalex.org/W1536128219","https://openalex.org/W1596062827","https://openalex.org/W1698199421","https://openalex.org/W1768625767","https://openalex.org/W1843246605","https://openalex.org/W1965335252","https://openalex.org/W1979142274","https://openalex.org/W1984187188","https://openalex.org/W1986463965","https://openalex.org/W1986721142","https://openalex.org/W1987453763","https://openalex.org/W2005295476","https://openalex.org/W2006546769","https://openalex.org/W2020061404","https://openalex.org/W2020465553","https://openalex.org/W2020923252","https://openalex.org/W2023754051","https://openalex.org/W2028211630","https://openalex.org/W2032864034","https://openalex.org/W2037557484","https://openalex.org/W2044781255","https://openalex.org/W2058534885","https://openalex.org/W2077576892","https://openalex.org/W2086219352","https://openalex.org/W2090789917","https://openalex.org/W2096563449","https://openalex.org/W2097708046","https://openalex.org/W2102675242","https://openalex.org/W2127143236","https://openalex.org/W2143628338","https://openalex.org/W2153287075","https://openalex.org/W2155203104","https://openalex.org/W2159322587","https://openalex.org/W2164136002","https://openalex.org/W2168575967","https://openalex.org/W2184104551","https://openalex.org/W2242842357","https://openalex.org/W2334575815","https://openalex.org/W2399506273","https://openalex.org/W2525579820","https://openalex.org/W2527516663","https://openalex.org/W2747290656","https://openalex.org/W2752099845","https://openalex.org/W2752524112","https://openalex.org/W2768281947","https://openalex.org/W2792200862","https://openalex.org/W2795968727","https://openalex.org/W2910385168","https://openalex.org/W2950577125","https://openalex.org/W2963752051","https://openalex.org/W2964212609","https://openalex.org/W2998387619","https://openalex.org/W3002406643","https://openalex.org/W3008478051","https://openalex.org/W3018494387","https://openalex.org/W3046802871","https://openalex.org/W3092494901","https://openalex.org/W3097483922","https://openalex.org/W3102702371","https://openalex.org/W3108144518","https://openalex.org/W3121261193","https://openalex.org/W4211165067","https://openalex.org/W4242732483","https://openalex.org/W4292157289","https://openalex.org/W4292887282","https://openalex.org/W6607133921","https://openalex.org/W6817713987"],"related_works":["https://openalex.org/W2593155302","https://openalex.org/W2041415459","https://openalex.org/W2072812638","https://openalex.org/W3126095231","https://openalex.org/W2086338133","https://openalex.org/W2352855287","https://openalex.org/W4200071106","https://openalex.org/W2378906650","https://openalex.org/W2384288445","https://openalex.org/W2382526412"],"abstract_inverted_index":{"Modern":[0],"machine":[1,132],"learning":[2,133],"and":[3,40,134,140,162,230],"computer":[4,135],"science":[5,136],"conferences":[6,47],"are":[7],"experiencing":[8],"a":[9,32,78,96,111,116,123,142,186,191,201,220],"surge":[10],"in":[11,81,130],"the":[12,18,24,42,57,75,100,103,127,148,152,241],"number":[13,25],"of":[14,20,26,61,118,126,151,165],"submissions":[15],"that":[16,102,177,234],"challenges":[17],"quality":[19,240],"peer":[21],"review":[22,106],"as":[23],"competent":[27],"reviewers":[28,94,120,157,178,214,236],"is":[29],"growing":[30],"at":[31,110,223],"much":[33],"slower":[34],"rate.":[35],"To":[36],"curb":[37],"this":[38,86,89],"trend":[39],"reduce":[41],"burden":[43],"on":[44,115,200],"reviewers,":[45],"several":[46],"have":[48,66],"started":[49],"encouraging":[50],"or":[51],"even":[52],"requiring":[53],"authors":[54],"to":[55,238],"declare":[56],"previous":[58],"submission":[59,104],"history":[60],"their":[62],"papers.":[63,173],"Such":[64],"initiatives":[65],"been":[67],"met":[68],"with":[69,155],"skepticism":[70],"among":[71],"authors,":[72],"who":[73,121,215],"raise":[74],"concern":[76],"about":[77,188],"potential":[79],"bias":[80,97],"reviewers'":[82],"recommendations":[83],"induced":[84],"by":[85,99],"information.":[87],"In":[88],"work,":[90],"we":[91,232],"investigate":[92],"whether":[93],"exhibit":[95],"caused":[98],"knowledge":[101],"under":[105],"was":[107],"previously":[108],"rejected":[109],"similar":[112],"venue,":[113],"focusing":[114],"population":[117],"novice":[119,235],"constitute":[122],"large":[124],"fraction":[125],"reviewer":[128],"pool":[129],"leading":[131],"conferences.":[137],"We":[138],"design":[139],"conduct":[141],"randomized":[143],"controlled":[144],"trial":[145],"closely":[146],"replicating":[147],"relevant":[149],"components":[150],"peer-review":[153],"pipeline":[154],"$133$":[156],"(master's,":[158],"junior":[159],"PhD":[160],"students,":[161],"recent":[163],"graduates":[164],"top":[166],"US":[167],"universities)":[168],"writing":[169],"reviews":[170],"for":[171],"$19$":[172],"The":[174],"analysis":[175],"reveals":[176],"indeed":[179],"become":[180],"negatively":[181],"biased":[182],"when":[183],"they":[184],"receive":[185,218],"signal":[187],"paper":[189],"being":[190],"resubmission,":[192],"giving":[193],"almost":[194],"1":[195],"point":[196],"lower":[197],"overall":[198],"score":[199],"10-point":[202],"Likert":[203],"item":[204],"(\u0394":[205],"=":[206,210],"-0.78,":[207],"95%":[208],"CI":[209],"[-1.30,":[211],"-0.24])":[212],"than":[213],"do":[216],"not":[217],"such":[219],"signal.":[221],"Looking":[222],"specific":[224],"criteria":[225],"scores":[226],"(originality,":[227],"quality,":[228],"clarity":[229],"significance),":[231],"observe":[233],"tend":[237],"underrate":[239],"most.":[242]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":3}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}