{"id":"https://openalex.org/W3187437888","doi":"https://doi.org/10.1145/3476076","title":"Goldilocks: Consistent Crowdsourced Scalar Annotations with Relative Uncertainty","display_name":"Goldilocks: Consistent Crowdsourced Scalar Annotations with Relative Uncertainty","publication_year":2021,"publication_date":"2021-10-13","ids":{"openalex":"https://openalex.org/W3187437888","doi":"https://doi.org/10.1145/3476076","mag":"3187437888"},"language":"en","primary_location":{"id":"doi:10.1145/3476076","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3476076","pdf_url":null,"source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2108.01799","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Quan Ze Chen","orcid":null},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Quan Ze Chen","raw_affiliation_strings":["University of Washington, Seattle, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Washington, Seattle, WA, USA","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Daniel S. Weld","orcid":null},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Daniel S. Weld","raw_affiliation_strings":["University of Washington, Seattle, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Washington, Seattle, WA, USA","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"last","author":{"id":null,"display_name":"Amy X. Zhang","orcid":null},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Amy X. Zhang","raw_affiliation_strings":["University of Washington, Seattle, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Washington, Seattle, WA, USA","institution_ids":["https://openalex.org/I201448701"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I201448701"],"apc_list":null,"apc_paid":null,"fwci":1.1536,"has_fulltext":false,"cited_by_count":9,"citation_normalized_percentile":{"value":0.82322919,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":"5","issue":"CSCW2","first_page":"1","last_page":"25"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9172000288963318,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9172000288963318,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.020899999886751175,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.005100000184029341,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/ambiguity","display_name":"Ambiguity","score":0.7739999890327454},{"id":"https://openalex.org/keywords/pairwise-comparison","display_name":"Pairwise comparison","score":0.6657999753952026},{"id":"https://openalex.org/keywords/bounding-overwatch","display_name":"Bounding overwatch","score":0.6376000046730042},{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.5756000280380249},{"id":"https://openalex.org/keywords/range","display_name":"Range (aeronautics)","score":0.46810001134872437},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.396699994802475},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.37720000743865967},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.35499998927116394},{"id":"https://openalex.org/keywords/resource","display_name":"Resource (disambiguation)","score":0.33719998598098755}],"concepts":[{"id":"https://openalex.org/C2780522230","wikidata":"https://www.wikidata.org/wiki/Q1140419","display_name":"Ambiguity","level":2,"score":0.7739999890327454},{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.6657999753952026},{"id":"https://openalex.org/C63584917","wikidata":"https://www.wikidata.org/wiki/Q333286","display_name":"Bounding overwatch","level":2,"score":0.6376000046730042},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.5756000280380249},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.553600013256073},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.48410001397132874},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4690999984741211},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.46810001134872437},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.40149998664855957},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.396699994802475},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.37720000743865967},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.35499998927116394},{"id":"https://openalex.org/C206345919","wikidata":"https://www.wikidata.org/wiki/Q20380951","display_name":"Resource (disambiguation)","level":2,"score":0.33719998598098755},{"id":"https://openalex.org/C137209882","wikidata":"https://www.wikidata.org/wiki/Q1403517","display_name":"Measurement uncertainty","level":2,"score":0.3246000111103058},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.31040000915527344},{"id":"https://openalex.org/C95623464","wikidata":"https://www.wikidata.org/wiki/Q1096149","display_name":"Classifier (UML)","level":2,"score":0.30979999899864197},{"id":"https://openalex.org/C197947376","wikidata":"https://www.wikidata.org/wiki/Q5155608","display_name":"Comparability","level":2,"score":0.3084999918937683},{"id":"https://openalex.org/C165838908","wikidata":"https://www.wikidata.org/wiki/Q736777","display_name":"Calibration","level":2,"score":0.29829999804496765},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.2955000102519989},{"id":"https://openalex.org/C57691317","wikidata":"https://www.wikidata.org/wiki/Q1289248","display_name":"Scalar (mathematics)","level":2,"score":0.2921000123023987},{"id":"https://openalex.org/C527412718","wikidata":"https://www.wikidata.org/wiki/Q855395","display_name":"Interpretation (philosophy)","level":2,"score":0.28279998898506165},{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.2815000116825104},{"id":"https://openalex.org/C106301342","wikidata":"https://www.wikidata.org/wiki/Q4117933","display_name":"Entropy (arrow of time)","level":2,"score":0.2791000008583069},{"id":"https://openalex.org/C2779545769","wikidata":"https://www.wikidata.org/wiki/Q5135364","display_name":"Closeness","level":2,"score":0.272599995136261},{"id":"https://openalex.org/C72161134","wikidata":"https://www.wikidata.org/wiki/Q5421219","display_name":"Expert elicitation","level":2,"score":0.26989999413490295},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.2538999915122986}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3476076","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3476076","pdf_url":null,"source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2108.01799","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2108.01799","pdf_url":"https://arxiv.org/pdf/2108.01799","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2108.01799","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2108.01799","pdf_url":"https://arxiv.org/pdf/2108.01799","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":45,"referenced_works":["https://openalex.org/W9014458","https://openalex.org/W12634471","https://openalex.org/W1702329404","https://openalex.org/W1962580118","https://openalex.org/W1973598714","https://openalex.org/W1988511622","https://openalex.org/W2021556168","https://openalex.org/W2067906953","https://openalex.org/W2105246029","https://openalex.org/W2106818711","https://openalex.org/W2109740547","https://openalex.org/W2133887027","https://openalex.org/W2134441771","https://openalex.org/W2162409443","https://openalex.org/W2290009368","https://openalex.org/W2292054779","https://openalex.org/W2398018337","https://openalex.org/W2400269077","https://openalex.org/W2411804288","https://openalex.org/W2509490062","https://openalex.org/W2540646130","https://openalex.org/W2574781439","https://openalex.org/W2599032673","https://openalex.org/W2607311634","https://openalex.org/W2610525153","https://openalex.org/W2782290562","https://openalex.org/W2798706400","https://openalex.org/W2807850020","https://openalex.org/W2885321846","https://openalex.org/W2888159079","https://openalex.org/W2896881735","https://openalex.org/W2898349875","https://openalex.org/W2899134503","https://openalex.org/W2902799923","https://openalex.org/W2944944282","https://openalex.org/W2954365773","https://openalex.org/W2962852048","https://openalex.org/W2980710694","https://openalex.org/W2987755095","https://openalex.org/W2997501826","https://openalex.org/W3035729454","https://openalex.org/W3080732897","https://openalex.org/W3105009080","https://openalex.org/W3163078977","https://openalex.org/W4391205406"],"related_works":[],"abstract_inverted_index":{"Human":[0],"ratings":[1],"have":[2],"become":[3],"a":[4,54,86,92],"crucial":[5],"resource":[6],"for":[7,18,60,94],"training":[8],"and":[9,20,28,40,84,117,141],"evaluating":[10],"machine":[11],"learning":[12],"systems.":[13],"However,":[14],"traditional":[15],"elicitation":[16,58],"methods":[17],"absolute":[19,79],"comparative":[21],"rating":[22,57,80],"suffer":[23],"from":[24,70],"issues":[25],"with":[26,82,146],"consistency":[27,130],"often":[29],"do":[30],"not":[31,139],"distinguish":[32],"between":[33,38],"uncertainty":[34,155],"due":[35],"to":[36,43,90,157],"disagreement":[37],"annotators":[39],"ambiguity":[41,69],"inherent":[42,68],"the":[44,136],"item":[45],"being":[46],"rated.":[47],"In":[48],"this":[49],"work,":[50],"we":[51],"present":[52],"Goldilocks,":[53],"novel":[55],"crowd":[56],"technique":[59],"collecting":[61],"calibrated":[62],"scalar":[63],"annotations":[64],"that":[65,125,142],"also":[66],"distinguishes":[67],"inter-annotator":[71],"disagreement.":[72],"We":[73,98,123],"introduce":[74],"two":[75],"main":[76],"ideas:":[77],"grounding":[78],"scales":[81],"examples":[83],"using":[85],"two-step":[87],"bounding":[88],"process":[89],"establish":[91],"range":[93],"an":[95],"item's":[96],"placement.":[97],"test":[99],"our":[100],"designs":[101],"in":[102,115,131],"three":[103],"domains:":[104],"judging":[105],"toxicity":[106],"of":[107,112,135,154,160],"online":[108],"comments,":[109],"estimating":[110,118],"satiety":[111],"food":[113],"depicted":[114],"images,":[116],"age":[119],"based":[120],"on":[121],"portraits.":[122],"show":[124],"(1)":[126],"Goldilocks":[127],"can":[128],"improve":[129],"domains":[132],"where":[133],"interpretation":[134],"scale":[137],"is":[138],"universal,":[140],"(2)":[143],"representing":[144],"items":[145],"ranges":[147],"lets":[148],"us":[149],"simultaneously":[150],"capture":[151],"different":[152],"sources":[153],"leading":[156],"better":[158],"estimates":[159],"pairwise":[161],"relationship":[162],"distributions.":[163]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":3},{"year":2022,"cited_by_count":1}],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2021-08-16T00:00:00"}