{"id":"https://openalex.org/W4387329169","doi":"https://doi.org/10.1145/3610074","title":"Judgment Sieve: Reducing Uncertainty in Group Judgments through Interventions Targeting Ambiguity versus Disagreement","display_name":"Judgment Sieve: Reducing Uncertainty in Group Judgments through Interventions Targeting Ambiguity versus Disagreement","publication_year":2023,"publication_date":"2023-09-28","ids":{"openalex":"https://openalex.org/W4387329169","doi":"https://doi.org/10.1145/3610074"},"language":"en","primary_location":{"id":"doi:10.1145/3610074","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3610074","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3610074","source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3610074","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5037054163","display_name":"Quan Ze Chen","orcid":null},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Quan Ze Chen","raw_affiliation_strings":["University of Washington, Seattle, WA, USA"],"raw_orcid":"https://orcid.org/0000-0002-6500-8922","affiliations":[{"raw_affiliation_string":"University of Washington, Seattle, WA, USA","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5037569091","display_name":"Amy X. Zhang","orcid":"https://orcid.org/0000-0001-9462-9835"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Amy X. Zhang","raw_affiliation_strings":["University of Washington, Seattle, WA, USA"],"raw_orcid":"https://orcid.org/0000-0001-9462-9835","affiliations":[{"raw_affiliation_string":"University of Washington, Seattle, WA, USA","institution_ids":["https://openalex.org/I201448701"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I201448701"],"apc_list":null,"apc_paid":null,"fwci":2.072,"has_fulltext":true,"cited_by_count":9,"citation_normalized_percentile":{"value":0.90047799,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":"7","issue":"CSCW2","first_page":"1","last_page":"26"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11147","display_name":"Misinformation and Its Impacts","score":0.9955000281333923,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9696999788284302,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/ambiguity","display_name":"Ambiguity","score":0.7881112098693848},{"id":"https://openalex.org/keywords/uncertainty-reduction-theory","display_name":"Uncertainty reduction theory","score":0.6972315311431885},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.6359904408454895},{"id":"https://openalex.org/keywords/deliberation","display_name":"Deliberation","score":0.5605524182319641},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5260821580886841},{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.4620510935783386},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.45156291127204895},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.42909687757492065},{"id":"https://openalex.org/keywords/intervention","display_name":"Intervention (counseling)","score":0.4217674136161804},{"id":"https://openalex.org/keywords/social-psychology","display_name":"Social psychology","score":0.3494637608528137},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.0966210663318634}],"concepts":[{"id":"https://openalex.org/C2780522230","wikidata":"https://www.wikidata.org/wiki/Q1140419","display_name":"Ambiguity","level":2,"score":0.7881112098693848},{"id":"https://openalex.org/C94361409","wikidata":"https://www.wikidata.org/wiki/Q7882500","display_name":"Uncertainty reduction theory","level":2,"score":0.6972315311431885},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.6359904408454895},{"id":"https://openalex.org/C2776946740","wikidata":"https://www.wikidata.org/wiki/Q358652","display_name":"Deliberation","level":3,"score":0.5605524182319641},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5260821580886841},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.4620510935783386},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.45156291127204895},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.42909687757492065},{"id":"https://openalex.org/C2780665704","wikidata":"https://www.wikidata.org/wiki/Q959298","display_name":"Intervention (counseling)","level":2,"score":0.4217674136161804},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.3494637608528137},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0966210663318634},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0},{"id":"https://openalex.org/C118552586","wikidata":"https://www.wikidata.org/wiki/Q7867","display_name":"Psychiatry","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3610074","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3610074","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3610074","source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1145/3610074","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3610074","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3610074","source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions","score":0.6899999976158142}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4387329169.pdf","grobid_xml":"https://content.openalex.org/works/W4387329169.grobid-xml"},"referenced_works_count":65,"referenced_works":["https://openalex.org/W9014458","https://openalex.org/W1838907055","https://openalex.org/W2026899120","https://openalex.org/W2036625304","https://openalex.org/W2042674443","https://openalex.org/W2069266605","https://openalex.org/W2071491733","https://openalex.org/W2081580037","https://openalex.org/W2098865355","https://openalex.org/W2105963649","https://openalex.org/W2108598243","https://openalex.org/W2111959010","https://openalex.org/W2117539524","https://openalex.org/W2150098611","https://openalex.org/W2173800084","https://openalex.org/W2290009368","https://openalex.org/W2295951612","https://openalex.org/W2563826943","https://openalex.org/W2574781439","https://openalex.org/W2599032673","https://openalex.org/W2605994771","https://openalex.org/W2607311634","https://openalex.org/W2613142859","https://openalex.org/W2614403205","https://openalex.org/W2726137762","https://openalex.org/W2752299099","https://openalex.org/W2771976988","https://openalex.org/W2783946743","https://openalex.org/W2796648593","https://openalex.org/W2896881735","https://openalex.org/W2898349875","https://openalex.org/W2898491875","https://openalex.org/W2899134503","https://openalex.org/W2917411097","https://openalex.org/W2946474696","https://openalex.org/W2985347336","https://openalex.org/W2985405314","https://openalex.org/W2987755095","https://openalex.org/W2988571156","https://openalex.org/W2991138171","https://openalex.org/W3023638732","https://openalex.org/W3031586727","https://openalex.org/W3034951181","https://openalex.org/W3049687886","https://openalex.org/W3103649165","https://openalex.org/W3109688857","https://openalex.org/W3123102563","https://openalex.org/W3124912334","https://openalex.org/W3163078977","https://openalex.org/W3166488533","https://openalex.org/W3170551880","https://openalex.org/W3183398589","https://openalex.org/W3187437888","https://openalex.org/W3205934028","https://openalex.org/W3207879356","https://openalex.org/W3212368439","https://openalex.org/W3216404684","https://openalex.org/W4206546829","https://openalex.org/W4236157893","https://openalex.org/W4280545690","https://openalex.org/W4288076433","https://openalex.org/W4288083802","https://openalex.org/W4293462114","https://openalex.org/W4301958211","https://openalex.org/W4306694236"],"related_works":["https://openalex.org/W2753773138","https://openalex.org/W3204354834","https://openalex.org/W6292469","https://openalex.org/W2801092833","https://openalex.org/W2901193592","https://openalex.org/W2118566385","https://openalex.org/W156167186","https://openalex.org/W2353179089","https://openalex.org/W2223796429","https://openalex.org/W2503756621"],"abstract_inverted_index":{"When":[0],"groups":[1],"of":[2,12,42,79,110,116,153,171,178,186,213],"people":[3],"are":[4],"tasked":[5],"with":[6,229],"making":[7],"a":[8,86,100,123,183,199],"judgment,":[9],"the":[10,28,43,54,76,163,168,191,206,233],"issue":[11],"uncertainty":[13,20,33,93,111,134,161,208,214,224],"often":[14],"arises.":[15],"Existing":[16],"methods":[17],"to":[18,48,57,75,91,130,217],"reduce":[19,92,133],"typically":[21],"focus":[22],"on":[23,135,144],"iteratively":[24],"improving":[25],"specificity":[26],"in":[27,94,99,222],"overall":[29],"task":[30],"instruction.":[31],"However,":[32],"can":[34,120],"arise":[35],"from":[36,225],"multiple":[37],"sources,":[38],"such":[39],"as":[40,215],"ambiguity":[41,176],"item":[44,137],"being":[45,138],"judged":[46],"due":[47,56],"limited":[49],"context,":[50],"or":[51,128],"disagreements":[52],"among":[53],"participants":[55],"different":[58,108],"perspectives":[59],"and":[60,151,180,182,188],"an":[61,113,175,230],"under-specified":[62],"task.":[63],"A":[64],"one-size-fits-all":[65],"intervention":[66,125],"may":[67],"be":[68],"ineffective":[69],"if":[70],"it":[71],"is":[72],"not":[73],"targeted":[74,101,124,158,203],"right":[77],"source":[78,227],"uncertainty.":[80],"In":[81,167],"this":[82],"paper":[83],"we":[84,119,173],"introduce":[85],"new":[87],"workflow,":[88],"Judgment":[89],"Sieve,":[90],"tasks":[95,193],"involving":[96],"group":[97],"judgment":[98,117],"manner.":[102],"By":[103],"utilizing":[104],"measurements":[105],"that":[106,157,201],"separate":[107],"sources":[109,212],"during":[112],"initial":[114],"round":[115],"elicitation,":[118],"then":[121],"select":[122],"adding":[126],"context":[127],"deliberation":[129],"most":[131,164],"effectively":[132],"each":[136],"judged.":[139],"We":[140,195],"test":[141],"our":[142,202],"approach":[143,204],"two":[145,192],"tasks:":[146],"rating":[147],"word":[148],"pair":[149],"similarity":[150],"toxicity":[152],"online":[154],"comments,":[155],"showing":[156],"interventions":[159],"reduced":[160,205],"for":[162,190,210,232],"uncertain":[165],"cases.":[166],"top":[169],"10%":[170],"cases,":[172],"saw":[174],"reduction":[177,185],"21.4%":[179],"25.7%,":[181],"disagreement":[184],"22.2%":[187],"11.2%":[189],"respectively.":[194],"also":[196],"found":[197],"through":[198],"simulation":[200],"average":[207,223],"scores":[209],"both":[211],"opposed":[216],"uniform":[218],"approaches":[219],"where":[220],"reductions":[221],"one":[226],"came":[228],"increase":[231],"other.":[234]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":1}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}