{"id":"https://openalex.org/W2800969132","doi":"https://doi.org/10.1145/3186195","title":"Further Insights on Drawing Sound Conclusions from Noisy Judgments","display_name":"Further Insights on Drawing Sound Conclusions from Noisy Judgments","publication_year":2018,"publication_date":"2018-04-30","ids":{"openalex":"https://openalex.org/W2800969132","doi":"https://doi.org/10.1145/3186195","mag":"2800969132"},"language":"en","primary_location":{"id":"doi:10.1145/3186195","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3186195","pdf_url":null,"source":{"id":"https://openalex.org/S4394735545","display_name":"ACM Transactions on Information Systems","issn_l":"1046-8188","issn":["1046-8188","1558-2868"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Information Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5003387956","display_name":"David M. Goldberg","orcid":"https://orcid.org/0000-0003-2617-5143"},"institutions":[{"id":"https://openalex.org/I4210150719","display_name":"eBay (United States)","ror":"https://ror.org/05cnabr44","country_code":"US","type":"company","lineage":["https://openalex.org/I4210150719"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"David Goldberg","raw_affiliation_strings":["eBay, California, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"eBay, California, USA","institution_ids":["https://openalex.org/I4210150719"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050903724","display_name":"Andrew Trotman","orcid":"https://orcid.org/0000-0003-1253-7123"},"institutions":[{"id":"https://openalex.org/I80281795","display_name":"University of Otago","ror":"https://ror.org/01jmxt844","country_code":"NZ","type":"education","lineage":["https://openalex.org/I80281795"]}],"countries":["NZ"],"is_corresponding":false,"raw_author_name":"Andrew Trotman","raw_affiliation_strings":["University of Otago, Dunedin, New Zealand"],"raw_orcid":"https://orcid.org/0000-0003-1253-7123","affiliations":[{"raw_affiliation_string":"University of Otago, Dunedin, New Zealand","institution_ids":["https://openalex.org/I80281795"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100411498","display_name":"Xiao Wang","orcid":"https://orcid.org/0000-0002-8652-8820"},"institutions":[{"id":"https://openalex.org/I4210150719","display_name":"eBay (United States)","ror":"https://ror.org/05cnabr44","country_code":"US","type":"company","lineage":["https://openalex.org/I4210150719"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xiao Wang","raw_affiliation_strings":["eBay, California, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"eBay, California, USA","institution_ids":["https://openalex.org/I4210150719"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019634451","display_name":"Wei Min","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wei Min","raw_affiliation_strings":["CreditX, Shanghai, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"CreditX, Shanghai, China","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5107030528","display_name":"Zongru Wan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zongru Wan","raw_affiliation_strings":["Evolution Labs, Shanghai, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Evolution Labs, Shanghai, China","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.9914,"has_fulltext":false,"cited_by_count":4,"citation_normalized_percentile":{"value":0.80419732,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":94},"biblio":{"volume":"36","issue":"4","first_page":"1","last_page":"31"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9824000000953674,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9768000245094299,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.8577749729156494},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8425899744033813},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.7700842618942261},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.6637793779373169},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.4965665936470032},{"id":"https://openalex.org/keywords/search-engine","display_name":"Search engine","score":0.4335036277770996},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.40423181653022766},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.39986473321914673},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.39177289605140686},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.11954385042190552}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.8577749729156494},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8425899744033813},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.7700842618942261},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.6637793779373169},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.4965665936470032},{"id":"https://openalex.org/C97854310","wikidata":"https://www.wikidata.org/wiki/Q19541","display_name":"Search engine","level":2,"score":0.4335036277770996},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.40423181653022766},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.39986473321914673},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.39177289605140686},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.11954385042190552},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3186195","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3186195","pdf_url":null,"source":{"id":"https://openalex.org/S4394735545","display_name":"ACM Transactions on Information Systems","issn_l":"1046-8188","issn":["1046-8188","1558-2868"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Information Systems","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":34,"referenced_works":["https://openalex.org/W9014458","https://openalex.org/W82775311","https://openalex.org/W108873425","https://openalex.org/W282666689","https://openalex.org/W1588207697","https://openalex.org/W1815682670","https://openalex.org/W1824124613","https://openalex.org/W1967370444","https://openalex.org/W1968927634","https://openalex.org/W1970381522","https://openalex.org/W1990190154","https://openalex.org/W2008285050","https://openalex.org/W2021581601","https://openalex.org/W2062236692","https://openalex.org/W2069870183","https://openalex.org/W2071323102","https://openalex.org/W2075893676","https://openalex.org/W2094607109","https://openalex.org/W2111310810","https://openalex.org/W2120308175","https://openalex.org/W2131676173","https://openalex.org/W2132232498","https://openalex.org/W2151592910","https://openalex.org/W2153763032","https://openalex.org/W2164777277","https://openalex.org/W2168071746","https://openalex.org/W2169585110","https://openalex.org/W2276504555","https://openalex.org/W2288846786","https://openalex.org/W2295951612","https://openalex.org/W2296739333","https://openalex.org/W2336806308","https://openalex.org/W2604385590","https://openalex.org/W2911978290"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W4384486036","https://openalex.org/W135177976","https://openalex.org/W1503094549","https://openalex.org/W2337920774","https://openalex.org/W2886410948","https://openalex.org/W2025875869","https://openalex.org/W4318823662","https://openalex.org/W1497983610","https://openalex.org/W1980158193"],"abstract_inverted_index":{"The":[0],"effectiveness":[1,59],"of":[2,18,25],"a":[3,92,164,172],"search":[4,124],"engine":[5,136],"is":[6,28,119],"typically":[7],"evaluated":[8],"using":[9,74],"hand-labeled":[10],"datasets,":[11],"where":[12,134],"the":[13,16,23,35,52,75,103,106,144,147,151,161],"labels":[14,26,42],"indicate":[15],"relevance":[17],"documents":[19],"to":[20,31,105,138,174],"queries.":[21],"Often":[22],"number":[24],"needed":[27],"too":[29],"large":[30],"be":[32,69,139],"created":[33],"by":[34,83,171],"best":[36],"annotators,":[37],"and":[38,54,64,95,98,169],"so":[39],"less":[40],"expensive":[41],"(e.g.,":[43],"from":[44],"crowdsourcing)":[45],"are":[46,126],"used.":[47],"This":[48,118],"introduces":[49],"errors":[50,56,67],"in":[51,57],"labels,":[53],"thus":[55],"standard":[58],"metrics":[60,104,152],"(such":[61],"as":[62],"P@k":[63],"DCG).":[65],"These":[66],"must":[68],"taken":[70],"into":[71],"consideration":[72],"when":[73,122],"metrics.":[76,130],"Previous":[77],"work":[78],"has":[79],"approached":[80],"assessor":[81],"error":[82],"taking":[84],"aggregates":[85],"over":[86],"multiple":[87],"inexpensive":[88],"assessors.":[89],"We":[90,131],"take":[91],"different":[93],"approach":[94],"introduce":[96],"equations":[97],"algorithms":[99],"that":[100],"can":[101],"adjust":[102],"values":[107],"they":[108],"would":[109],"have":[110],"had":[111],"if":[112],"there":[113],"were":[114,153],"no":[115],"annotation":[116,156,177],"errors.":[117],"especially":[120],"important":[121],"two":[123],"engines":[125],"compared":[127],"on":[128],"their":[129],"give":[132],"examples":[133],"one":[135],"appeared":[137],"statistically":[140],"significantly":[141],"better":[142],"than":[143],"other,":[145],"but":[146],"effect":[148],"disappeared":[149],"after":[150],"corrected":[154],"for":[155,176],"error.":[157,178],"In":[158],"other":[159],"words,":[160],"evidence":[162],"supporting":[163],"statistical":[165],"difference":[166],"was":[167],"illusory":[168],"caused":[170],"failure":[173],"account":[175]},"counts_by_year":[{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":1},{"year":2018,"cited_by_count":1}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}