{"id":"https://openalex.org/W6987787487","doi":null,"title":"Understanding and Addressing Uncertainty of the Crowd","display_name":"Understanding and Addressing Uncertainty of the Crowd","alternate_titles":[],"publication_year":2022,"publication_date":"2022-01-01","ids":{"openalex":"https://openalex.org/W6987787487"},"language":"en","primary_location":{"id":"pmh:oai:digital.lib.washington.edu:1773/49879","doi":null,"is_oa":true,"landing_page_url":"http://hdl.handle.net/1773/49879","pdf_url":"https://digital.lib.washington.edu/bitstreams/6f1df795-91c5-49c1-b3c8-d797407dd57e/download","source":{"id":"https://openalex.org/S4306401128","display_name":"ResearchWorks at the University of Washington (University of Washington)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I201448701","host_organization_name":"University of Washington","host_organization_lineage":["https://openalex.org/I201448701"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Thesis"},"parent_works":[],"lineage":["https://openalex.org/W6987787487"],"child_works_count":0,"type":"dissertation","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://digital.lib.washington.edu/bitstreams/6f1df795-91c5-49c1-b3c8-d797407dd57e/download","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Chen, Quan Ze","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":true,"contributor_roles":[],"raw_author_name":"Chen, Quan Ze","raw_affiliation_strings":["Computer Science & Engineering","Computer Science & Engineering, University of Washington"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Computer Science & Engineering","institution_ids":[]},{"raw_affiliation_string":"Computer Science & Engineering, University of Washington","institution_ids":["https://openalex.org/I201448701"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I201448701"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":true,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9843999743461609,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9843999743461609,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.0019000000320374966,"subfield":{"id":"https://openalex.org/subfields/3308","display_name":"Law"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.0012000000569969416,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"crowdsourcing","score":0.8330000042915344},{"id":"https://openalex.org/keywords/annotation-uncertainty","display_name":"annotation uncertainty","score":0.43700000643730164},{"id":"https://openalex.org/keywords/precedent","display_name":"precedent","score":0.6859999895095825},{"id":"https://openalex.org/keywords/categorical-classification","display_name":"categorical classification","score":0.5419999957084656},{"id":"https://openalex.org/keywords/disagreement","display_name":"disagreement","score":0.5170000195503235},{"id":"https://openalex.org/keywords/cicero","display_name":"Cicero","score":0.7620000243186951},{"id":"https://openalex.org/keywords/judgment-under-uncertainty","display_name":"judgment under uncertainty","score":0.6600000262260437},{"id":"https://openalex.org/keywords/annotation-consistency","display_name":"annotation consistency","score":0.4300000071525574}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.46540001034736633},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.2912999987602234},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.290800005197525},{"id":"https://openalex.org/C2778137410","wikidata":"https://www.wikidata.org/wiki/Q2732820","display_name":"Government (linguistics)","level":2,"score":0.2775999903678894},{"id":"https://openalex.org/C108170787","wikidata":"https://www.wikidata.org/wiki/Q3951828","display_name":"Agency (philosophy)","level":2,"score":0.2750000059604645},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.27459999918937683},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.2718999981880188},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.2621000111103058},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.2599000036716461},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.25920000672340393},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.25769999623298645},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.2572999894618988},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.25279998779296875}],"mesh":[],"locations_count":1,"locations":[{"id":"pmh:oai:digital.lib.washington.edu:1773/49879","doi":null,"is_oa":true,"landing_page_url":"http://hdl.handle.net/1773/49879","pdf_url":"https://digital.lib.washington.edu/bitstreams/6f1df795-91c5-49c1-b3c8-d797407dd57e/download","source":{"id":"https://openalex.org/S4306401128","display_name":"ResearchWorks at the University of Washington (University of Washington)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I201448701","host_organization_name":"University of Washington","host_organization_lineage":["https://openalex.org/I201448701"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Thesis"}],"best_oa_location":{"id":"pmh:oai:digital.lib.washington.edu:1773/49879","doi":null,"is_oa":true,"landing_page_url":"http://hdl.handle.net/1773/49879","pdf_url":"https://digital.lib.washington.edu/bitstreams/6f1df795-91c5-49c1-b3c8-d797407dd57e/download","source":{"id":"https://openalex.org/S4306401128","display_name":"ResearchWorks at the University of Washington (University of Washington)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I201448701","host_organization_name":"University of Washington","host_organization_lineage":["https://openalex.org/I201448701"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Thesis"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W6987787487.pdf","grobid_xml":"https://content.openalex.org/works/W6987787487.grobid-xml"},"referenced_works_count":263,"referenced_works":["https://openalex.org/W7271355","https://openalex.org/W9014458","https://openalex.org/W12634471","https://openalex.org/W279631868","https://openalex.org/W588472770","https://openalex.org/W621249151","https://openalex.org/W1185009508","https://openalex.org/W1507711477","https://openalex.org/W1520232900","https://openalex.org/W1538874948","https://openalex.org/W1552365889","https://openalex.org/W1562120320","https://openalex.org/W1597892640","https://openalex.org/W1702329404","https://openalex.org/W1817172274","https://openalex.org/W1838907055","https://openalex.org/W1945616565","https://openalex.org/W1962580118","https://openalex.org/W1968115966","https://openalex.org/W1970381522","https://openalex.org/W1973598714","https://openalex.org/W1986649315","https://openalex.org/W1988511622","https://openalex.org/W1990584714","https://openalex.org/W1999308248","https://openalex.org/W2008236228","https://openalex.org/W2009132821","https://openalex.org/W2015650888","https://openalex.org/W2017803312","https://openalex.org/W2019566340","https://openalex.org/W2019843401","https://openalex.org/W2021556168","https://openalex.org/W2025720061","https://openalex.org/W2026899120","https://openalex.org/W2028953510","https://openalex.org/W2033442452","https://openalex.org/W2036625304","https://openalex.org/W2042674443","https://openalex.org/W2047956997","https://openalex.org/W2049248196","https://openalex.org/W2053921957","https://openalex.org/W2058556535","https://openalex.org/W2063782051","https://openalex.org/W2067906953","https://openalex.org/W2069266605","https://openalex.org/W2071491733","https://openalex.org/W2075585362","https://openalex.org/W2076649972","https://openalex.org/W2078429476","https://openalex.org/W2081580037","https://openalex.org/W2088981844","https://openalex.org/W2098865355","https://openalex.org/W2099531266","https://openalex.org/W2102267996","https://openalex.org/W2105246029","https://openalex.org/W2105963649","https://openalex.org/W2106818711","https://openalex.org/W2108598243","https://openalex.org/W2109740547","https://openalex.org/W2111298664","https://openalex.org/W2111959010","https://openalex.org/W2112496463","https://openalex.org/W2113878109","https://openalex.org/W2117539524","https://openalex.org/W2120396827","https://openalex.org/W2122429065","https://openalex.org/W2127008633","https://openalex.org/W2131723483","https://openalex.org/W2133887027","https://openalex.org/W2134441771","https://openalex.org/W2142518823","https://openalex.org/W2146006411","https://openalex.org/W2148479118","https://openalex.org/W2150098611","https://openalex.org/W2153332911","https://openalex.org/W2155243985","https://openalex.org/W2157289187","https://openalex.org/W2162409443","https://openalex.org/W2164019165","https://openalex.org/W2168765606","https://openalex.org/W2173800084","https://openalex.org/W2207487101","https://openalex.org/W2250638193","https://openalex.org/W2251089262","https://openalex.org/W2251832915","https://openalex.org/W2251847161","https://openalex.org/W2252166243","https://openalex.org/W2290009368","https://openalex.org/W2292054779","https://openalex.org/W2294447324","https://openalex.org/W2307292283","https://openalex.org/W2395172628","https://openalex.org/W2398018337","https://openalex.org/W2400269077","https://openalex.org/W2400672603","https://openalex.org/W2411804288","https://openalex.org/W2460077910","https://openalex.org/W2467989257","https://openalex.org/W2469104253","https://openalex.org/W2509490062","https://openalex.org/W2540646130","https://openalex.org/W2554839354","https://openalex.org/W2558888286","https://openalex.org/W2559661079","https://openalex.org/W2560267416","https://openalex.org/W2563826943","https://openalex.org/W2574781439","https://openalex.org/W2585712495","https://openalex.org/W2598764358","https://openalex.org/W2599032673","https://openalex.org/W2602196754","https://openalex.org/W2605994771","https://openalex.org/W2606766508","https://openalex.org/W2607311634","https://openalex.org/W2610525153","https://openalex.org/W2613142859","https://openalex.org/W2614403205","https://openalex.org/W2626967530","https://openalex.org/W2726137762","https://openalex.org/W2741649465","https://openalex.org/W2752299099","https://openalex.org/W2759211898","https://openalex.org/W2782290562","https://openalex.org/W2783668259","https://openalex.org/W2783946743","https://openalex.org/W2787471476","https://openalex.org/W2788481061","https://openalex.org/W2794219599","https://openalex.org/W2794424581","https://openalex.org/W2795038878","https://openalex.org/W2795596920","https://openalex.org/W2796648593","https://openalex.org/W2797977484","https://openalex.org/W2798706400","https://openalex.org/W2807524009","https://openalex.org/W2807850020","https://openalex.org/W2810228847","https://openalex.org/W2888159079","https://openalex.org/W2890059736","https://openalex.org/W2892237532","https://openalex.org/W2896881735","https://openalex.org/W2897210067","https://openalex.org/W2898349875","https://openalex.org/W2898491875","https://openalex.org/W2898911770","https://openalex.org/W2898944906","https://openalex.org/W2898989642","https://openalex.org/W2899027170","https://openalex.org/W2899134503","https://openalex.org/W2899164549","https://openalex.org/W2899346237","https://openalex.org/W2899349505","https://openalex.org/W2902799923","https://openalex.org/W2911227954","https://openalex.org/W2913303681","https://openalex.org/W2917411097","https://openalex.org/W2918080118","https://openalex.org/W2928915328","https://openalex.org/W2936262349","https://openalex.org/W2943845219","https://openalex.org/W2944870985","https://openalex.org/W2944944282","https://openalex.org/W2946474696","https://openalex.org/W2949678053","https://openalex.org/W2954365773","https://openalex.org/W2969282158","https://openalex.org/W2969489725","https://openalex.org/W2974290810","https://openalex.org/W2978017171","https://openalex.org/W2979893369","https://openalex.org/W2980710694","https://openalex.org/W2983361284","https://openalex.org/W2984353433","https://openalex.org/W2985347336","https://openalex.org/W2985405314","https://openalex.org/W2987755095","https://openalex.org/W2988571156","https://openalex.org/W2989364584","https://openalex.org/W2996426982","https://openalex.org/W2997654229","https://openalex.org/W2999637955","https://openalex.org/W3005013146","https://openalex.org/W3014596384","https://openalex.org/W3023638732","https://openalex.org/W3030681628","https://openalex.org/W3031586727","https://openalex.org/W3032574806","https://openalex.org/W3034363135","https://openalex.org/W3035371891","https://openalex.org/W3035729454","https://openalex.org/W3037013468","https://openalex.org/W3037634279","https://openalex.org/W3038897811","https://openalex.org/W3047049674","https://openalex.org/W3049687886","https://openalex.org/W3080732897","https://openalex.org/W3082276260","https://openalex.org/W3088599783","https://openalex.org/W3088647504","https://openalex.org/W3088965898","https://openalex.org/W3094043653","https://openalex.org/W3105009080","https://openalex.org/W3105111141","https://openalex.org/W3105700517","https://openalex.org/W3109688857","https://openalex.org/W3111557190","https://openalex.org/W3118813946","https://openalex.org/W3119067070","https://openalex.org/W3124432962","https://openalex.org/W3142359849","https://openalex.org/W3152824819","https://openalex.org/W3154945374","https://openalex.org/W3160168168","https://openalex.org/W3160676623","https://openalex.org/W3163078977","https://openalex.org/W3163469193","https://openalex.org/W3166488533","https://openalex.org/W3166716308","https://openalex.org/W3168006205","https://openalex.org/W3170551880","https://openalex.org/W3171875349","https://openalex.org/W3174249157","https://openalex.org/W3180717668","https://openalex.org/W3183398589","https://openalex.org/W3184649139","https://openalex.org/W3187437888","https://openalex.org/W3189849087","https://openalex.org/W3192957100","https://openalex.org/W3196190653","https://openalex.org/W3199811133","https://openalex.org/W3201395296","https://openalex.org/W3204854750","https://openalex.org/W3206428286","https://openalex.org/W3206621920","https://openalex.org/W3212331449","https://openalex.org/W3212896246","https://openalex.org/W3213728774","https://openalex.org/W3217570815","https://openalex.org/W4200270317","https://openalex.org/W4200503042","https://openalex.org/W4200632307","https://openalex.org/W4220897491","https://openalex.org/W4225006216","https://openalex.org/W4226092640","https://openalex.org/W4226107190","https://openalex.org/W4226412112","https://openalex.org/W4230006243","https://openalex.org/W4244408479","https://openalex.org/W4280617721","https://openalex.org/W4288076433","https://openalex.org/W4292157289","https://openalex.org/W4292341621","https://openalex.org/W4293221248","https://openalex.org/W4293462114","https://openalex.org/W4301347335","https://openalex.org/W4306694236","https://openalex.org/W4313459382","https://openalex.org/W4385573665","https://openalex.org/W4394634302","https://openalex.org/W6931300961","https://openalex.org/W6982535598","https://openalex.org/W7095463260","https://openalex.org/W7124161300"],"related_works":["https://openalex.org/W2908244850","https://openalex.org/W7011503813","https://openalex.org/W2589927650","https://openalex.org/W7055415530","https://openalex.org/W2601651011","https://openalex.org/W6990662315","https://openalex.org/W2539881874","https://openalex.org/W3022640235","https://openalex.org/W2784064574","https://openalex.org/W2117656218"],"abstract_inverted_index":{"Uncertainty":[0],"is":[1],"crucial":[2],"to":[3,13,24,35,37,47,67,80,141,228,236],"understanding":[4,211],"human":[5,120],"judgments.":[6,59,121],"Whether":[7],"it's":[8],"annotators":[9],"producing":[10],"data":[11],"used":[12],"train":[14],"and":[15,44,50,73,87,100,109,118,212,251,262],"evaluate":[16],"machine":[17],"learning":[18],"systems,":[19],"teaching":[20],"staff":[21],"assigning":[22],"grades":[23],"open-ended":[25],"student":[26],"responses,":[27],"or":[28],"online":[29],"communities":[30],"adjudicating":[31],"the":[32,52,61,78,176,238,246],"moderation":[33],"action":[34],"apply":[36],"a":[38,95,128,156,187,216,221,225],"piece":[39],"of":[40,63,70,97,113,139,178,194,240,248],"content,":[41],"many":[42],"groups":[43,72],"individuals":[45,74],"need":[46,79],"account":[48],"for":[49,130,210,255],"address":[51,110],"uncertainty":[53,114,140,164,195,214,230],"that":[54,102,135,158,189,219,233,264],"comes":[55],"along":[56],"with":[57,77],"making":[58],"As":[60],"application":[62],"computing":[64],"technology":[65],"expands":[66],"more":[68],"areas":[69],"society,":[71],"are":[75],"faced":[76],"make":[81],"judgments":[82],"on":[83,165,224],"increasingly":[84],"complex,":[85],"subjective,":[86],"nuanced":[88],"tasks":[89,170],"at":[90],"scale.":[91],"This":[92],"dissertation":[93],"presents":[94],"set":[96],"novel":[98],"tools":[99,209,250,261],"processes":[101,263],"improve":[103],"upon":[104],"how":[105],"we":[106],"capture,":[107],"distinguish,":[108],"various":[111],"sources":[112,138],"present":[115,185],"in":[116,175,258],"individual":[117,208],"collective":[119],"I":[122,149,183,204,242],"will":[123,150,184,205],"start":[124],"by":[125,171,244],"introducing,":[126],"Goldilocks,":[127],"tool":[129,188],"conducting":[131],"scalar":[132],"rating":[133],"annotations":[134],"enables":[136,159],"different":[137],"be":[142],"distinguished":[143],"while":[144],"also":[145],"improving":[146],"consistency.":[147],"Then":[148],"introduce":[151],"case":[152],"law":[153],"crowdsourcing":[154],"as":[155],"process":[157],"capturing":[160],"similar":[161],"insights":[162,254],"about":[163],"complex":[166],"categorical":[167],"classification":[168],"judgment":[169,267],"utilizing":[172],"prior":[173],"decisions":[174],"form":[177],"precedent":[179],"cases.":[180],"Following":[181],"this,":[182],"Cicero,":[186],"addresses":[190],"one":[191],"specific":[192],"source":[193,239],"--":[196,198],"disagreement":[197],"through":[199,215],"multi-turn,":[200],"contextual":[201],"deliberation.":[202],"Finally,":[203],"tie":[206],"together":[207],"addressing":[213],"dynamic":[217],"workflow":[218],"applies":[220],"targeted":[222],"intervention":[223],"per-instance":[226],"scale":[227],"reduce":[229],"using":[231],"measurements":[232],"allow":[234],"us":[235],"distinguish":[237],"uncertainty.":[241,269],"conclude":[243],"discussing":[245],"limitations":[247],"current":[249],"give":[252],"some":[253],"future":[256],"work":[257],"designing":[259],"new":[260],"natively":[265],"support":[266],"under":[268]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W6987787487","counts_by_year":[],"updated_date":"2026-11-20T13:11:24.684918","created_date":"2025-10-10T00:00:00"}
