{"id":"https://openalex.org/W2914980526","doi":"https://doi.org/10.1145/3308558.3313459","title":"Truth Inference at Scale: A Bayesian Model for Adjudicating Highly Redundant Crowd Annotations","display_name":"Truth Inference at Scale: A Bayesian Model for Adjudicating Highly Redundant Crowd Annotations","publication_year":2019,"publication_date":"2019-05-13","ids":{"openalex":"https://openalex.org/W2914980526","doi":"https://doi.org/10.1145/3308558.3313459","mag":"2914980526"},"language":"en","primary_location":{"id":"doi:10.1145/3308558.3313459","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3308558.3313459","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"The World Wide Web Conference","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3308558.3313459","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101841379","display_name":"Yuan Li","orcid":"https://orcid.org/0000-0001-9022-7345"},"institutions":[{"id":"https://openalex.org/I165779595","display_name":"University of Melbourne","ror":"https://ror.org/01ej9dk98","country_code":"AU","type":"education","lineage":["https://openalex.org/I165779595"]}],"countries":["AU"],"is_corresponding":true,"raw_author_name":"Yuan Li","raw_affiliation_strings":["The University of Melbourne, Australia"],"affiliations":[{"raw_affiliation_string":"The University of Melbourne, Australia","institution_ids":["https://openalex.org/I165779595"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078824132","display_name":"Benjamin I. P. Rubinstein","orcid":"https://orcid.org/0000-0002-2947-6980"},"institutions":[{"id":"https://openalex.org/I165779595","display_name":"University of Melbourne","ror":"https://ror.org/01ej9dk98","country_code":"AU","type":"education","lineage":["https://openalex.org/I165779595"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Benjamin I. P. Rubinstein","raw_affiliation_strings":["The University of Melbourne, Australia"],"affiliations":[{"raw_affiliation_string":"The University of Melbourne, Australia","institution_ids":["https://openalex.org/I165779595"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5078530959","display_name":"Trevor Cohn","orcid":"https://orcid.org/0000-0003-4363-1673"},"institutions":[{"id":"https://openalex.org/I165779595","display_name":"University of Melbourne","ror":"https://ror.org/01ej9dk98","country_code":"AU","type":"education","lineage":["https://openalex.org/I165779595"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Trevor Cohn","raw_affiliation_strings":["The University of Melbourne, Australia"],"affiliations":[{"raw_affiliation_string":"The University of Melbourne, Australia","institution_ids":["https://openalex.org/I165779595"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5101841379"],"corresponding_institution_ids":["https://openalex.org/I165779595"],"apc_list":null,"apc_paid":null,"fwci":5.6094,"has_fulltext":true,"cited_by_count":28,"citation_normalized_percentile":{"value":0.95539882,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"1028","last_page":"1038"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10764","display_name":"Privacy-Preserving Technologies in Data","score":0.9958000183105469,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9950000047683716,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.80361008644104},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.6867501139640808},{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.6262460350990295},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.5496802926063538},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5493316054344177},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.521176278591156},{"id":"https://openalex.org/keywords/bayesian-inference","display_name":"Bayesian inference","score":0.515128493309021},{"id":"https://openalex.org/keywords/prior-probability","display_name":"Prior probability","score":0.43767470121383667},{"id":"https://openalex.org/keywords/heuristic","display_name":"Heuristic","score":0.4331095218658447},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.43301889300346375},{"id":"https://openalex.org/keywords/bayesian-probability","display_name":"Bayesian probability","score":0.4001638889312744},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.3466501235961914}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.80361008644104},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.6867501139640808},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.6262460350990295},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.5496802926063538},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5493316054344177},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.521176278591156},{"id":"https://openalex.org/C160234255","wikidata":"https://www.wikidata.org/wiki/Q812535","display_name":"Bayesian inference","level":3,"score":0.515128493309021},{"id":"https://openalex.org/C177769412","wikidata":"https://www.wikidata.org/wiki/Q278090","display_name":"Prior probability","level":3,"score":0.43767470121383667},{"id":"https://openalex.org/C173801870","wikidata":"https://www.wikidata.org/wiki/Q201413","display_name":"Heuristic","level":2,"score":0.4331095218658447},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.43301889300346375},{"id":"https://openalex.org/C107673813","wikidata":"https://www.wikidata.org/wiki/Q812534","display_name":"Bayesian probability","level":2,"score":0.4001638889312744},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3466501235961914},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3308558.3313459","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3308558.3313459","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"The World Wide Web Conference","raw_type":"proceedings-article"},{"id":"pmh:oai:jupiter.its.unimelb.edu.au:11343/225739","is_oa":true,"landing_page_url":"http://gateway.webofknowledge.com/gateway/Gateway.cgi?GWVersion=2&SrcApp=PARTNER_APP&SrcAuth=LinksAMR&KeyUT=WOS:000483508401009&DestLinkType=FullRecord&DestApp=ALL_WOS&UsrCustomerID=d4d813f4571fa7d6246bdc0dfeca3a1c","pdf_url":"http://hdl.handle.net/11343/225739","source":{"id":"https://openalex.org/S4377196259","display_name":"Minerva Access (University of Melbourne)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I165779595","host_organization_name":"The University of Melbourne","host_organization_lineage":["https://openalex.org/I165779595"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"The World Wide Web Conference","raw_type":"Conference Paper"}],"best_oa_location":{"id":"doi:10.1145/3308558.3313459","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3308558.3313459","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"The World Wide Web Conference","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320306078","display_name":"U.S. Department of Defense","ror":"https://ror.org/0447fe631"},{"id":"https://openalex.org/F4320332180","display_name":"Defense Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"},{"id":"https://openalex.org/F4320334704","display_name":"Australian Research Council","ror":"https://ror.org/05mmh0f86"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":34,"referenced_works":["https://openalex.org/W9014458","https://openalex.org/W567643054","https://openalex.org/W1141062540","https://openalex.org/W1459599406","https://openalex.org/W1516111018","https://openalex.org/W1590411898","https://openalex.org/W1839751919","https://openalex.org/W1970381522","https://openalex.org/W2049633694","https://openalex.org/W2056748234","https://openalex.org/W2113878109","https://openalex.org/W2115915304","https://openalex.org/W2127849236","https://openalex.org/W2134305421","https://openalex.org/W2141649520","https://openalex.org/W2142518823","https://openalex.org/W2149273804","https://openalex.org/W2150612552","https://openalex.org/W2152009989","https://openalex.org/W2157010107","https://openalex.org/W2290431464","https://openalex.org/W2292361954","https://openalex.org/W2294447324","https://openalex.org/W2408493841","https://openalex.org/W2427527485","https://openalex.org/W2585226541","https://openalex.org/W2618530766","https://openalex.org/W2735557082","https://openalex.org/W2930957955","https://openalex.org/W2949312134","https://openalex.org/W2963748441","https://openalex.org/W3123895079","https://openalex.org/W6628785978","https://openalex.org/W6718053083"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W135177976","https://openalex.org/W4384486036","https://openalex.org/W1503094549","https://openalex.org/W2337920774","https://openalex.org/W4286908577","https://openalex.org/W2886410948","https://openalex.org/W2164129707","https://openalex.org/W4292122269","https://openalex.org/W2949366006"],"abstract_inverted_index":{"Crowd-sourcing":[0],"is":[1,77,115,183,205,207],"a":[2,44,155,160],"cheap":[3],"and":[4,10,88,124,144,167,182,215],"popular":[5],"means":[6],"of":[7,21,38,56,73,110,128,213],"creating":[8],"training":[9],"evaluation":[11],"datasets":[12,103],"for":[13],"machine":[14],"learning,":[15],"however":[16],"it":[17],"poses":[18],"the":[19,54,71,113,125,137,178,184,190],"problem":[20],"'truth":[22],"inference',":[23],"as":[24],"individual":[25],"workers":[26,85,129],"cannot":[27],"be":[28],"wholly":[29],"trusted":[30],"to":[31,42,52,177],"provide":[32],"reliable":[33],"annotations.":[34],"Research":[35],"into":[36],"models":[37,148],"annotation":[39],"aggregation":[40],"attempts":[41],"infer":[43],"latent":[45],"'true'":[46],"annotation,":[47],"which":[48],"has":[49],"been":[50],"shown":[51,198],"improve":[53],"utility":[55],"crowd-sourced":[57],"data.":[58],"However,":[59],"existing":[60],"techniques":[61],"beat":[62],"simple":[63,151,168],"baselines":[64],"only":[65,185,210],"in":[66,82,209,217],"low":[67,78,90],"redundancy":[68],"settings,":[69,136],"where":[70,84],"number":[72],"annotations":[74,92,121],"per":[75,122],"instance":[76],"(=":[79,119],"3),":[80],"or":[81,97],"situations":[83],"are":[86,107],"unreliable":[87],"produce":[89,130],"quality":[91,132],"(e.g.,":[93],"through":[94],"spamming,":[95],"random,":[96],"adversarial":[98],"behaviours.)":[99],"As":[100],"we":[101],"show,":[102],"produced":[104],"by":[105,199],"crowd-sourcing":[106],"often":[108],"not":[109],"this":[111,150],"type:":[112],"data":[114],"highly":[116],"redundantly":[117],"annotated":[118],"5":[120],"instance),":[123],"vast":[126],"majority":[127,138,191],"high":[131],"outputs.":[133],"In":[134],"these":[135],"vote":[139,192],"heuristic":[140,193],"performs":[141],"very":[142],"well,":[143],"most":[145],"truth":[146],"inference":[147],"underperform":[149],"baseline.":[152],"We":[153],"propose":[154],"novel":[156],"technique,":[157],"based":[158],"on":[159],"Bayesian":[161],"graphical":[162],"model":[163],"with":[164],"conjugate":[165],"priors,":[166],"iterative":[169],"expectation-maximisation":[170],"inference.":[171],"Our":[172],"technique":[173,204],"produces":[174],"competitive":[175],"performance":[176],"state-of-the-art":[179],"benchmark":[180],"methods,":[181],"method":[186],"that":[187],"significantly":[188],"outperforms":[189],"at":[194],"one-sided":[195],"level":[196],"0.025,":[197],"significance":[200],"tests.":[201],"Moreover,":[202],"our":[203],"simple,":[206],"implemented":[208],"50":[211],"lines":[212],"code,":[214],"trains":[216],"seconds.":[218],"1":[219]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":6},{"year":2021,"cited_by_count":9},{"year":2020,"cited_by_count":1},{"year":2019,"cited_by_count":2}],"updated_date":"2026-03-11T14:59:36.786465","created_date":"2025-10-10T00:00:00"}