{"id":"https://openalex.org/W2054593810","doi":"https://doi.org/10.1145/2723372.2723722","title":"The Importance of Being Expert","display_name":"The Importance of Being Expert","publication_year":2015,"publication_date":"2015-05-27","ids":{"openalex":"https://openalex.org/W2054593810","doi":"https://doi.org/10.1145/2723372.2723722","mag":"2054593810"},"language":"en","primary_location":{"id":"doi:10.1145/2723372.2723722","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2723372.2723722","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2015 ACM SIGMOD International Conference on Management of Data","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://iris.uniroma1.it/bitstream/11573/842138/7/Anagnostopoulos_postprint_The-Importance_2015.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5000949710","display_name":"Aris Anagnostopoulos","orcid":"https://orcid.org/0000-0001-9183-7911"},"institutions":[{"id":"https://openalex.org/I861853513","display_name":"Sapienza University of Rome","ror":"https://ror.org/02be6w209","country_code":"IT","type":"education","lineage":["https://openalex.org/I861853513"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Aris Anagnostopoulos","raw_affiliation_strings":["Sapienza University of Rome, Rome, Italy","Sapienza university of Rome, Rome, Italy#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Sapienza University of Rome, Rome, Italy","institution_ids":["https://openalex.org/I861853513"]},{"raw_affiliation_string":"Sapienza university of Rome, Rome, Italy#TAB#","institution_ids":["https://openalex.org/I861853513"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039929633","display_name":"Luca Becchetti","orcid":"https://orcid.org/0000-0002-4941-0532"},"institutions":[{"id":"https://openalex.org/I861853513","display_name":"Sapienza University of Rome","ror":"https://ror.org/02be6w209","country_code":"IT","type":"education","lineage":["https://openalex.org/I861853513"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Luca Becchetti","raw_affiliation_strings":["Sapienza University of Rome, Rome, Italy","Sapienza university of Rome, Rome, Italy#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Sapienza University of Rome, Rome, Italy","institution_ids":["https://openalex.org/I861853513"]},{"raw_affiliation_string":"Sapienza university of Rome, Rome, Italy#TAB#","institution_ids":["https://openalex.org/I861853513"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5073663860","display_name":"Adriano Fazzone","orcid":"https://orcid.org/0000-0002-3062-4485"},"institutions":[{"id":"https://openalex.org/I861853513","display_name":"Sapienza University of Rome","ror":"https://ror.org/02be6w209","country_code":"IT","type":"education","lineage":["https://openalex.org/I861853513"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Adriano Fazzone","raw_affiliation_strings":["Sapienza University of Rome, Rome, Italy","Sapienza university of Rome, Rome, Italy#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Sapienza University of Rome, Rome, Italy","institution_ids":["https://openalex.org/I861853513"]},{"raw_affiliation_string":"Sapienza university of Rome, Rome, Italy#TAB#","institution_ids":["https://openalex.org/I861853513"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004545243","display_name":"Ida Mele","orcid":"https://orcid.org/0000-0002-3730-6383"},"institutions":[{"id":"https://openalex.org/I4210109712","display_name":"Max Planck Institute for Informatics","ror":"https://ror.org/01w19ak89","country_code":"DE","type":"facility","lineage":["https://openalex.org/I149899117","https://openalex.org/I4210109712"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Ida Mele","raw_affiliation_strings":["Max Planck Institute for Informatics, Saarbr\u00fccken, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Max Planck Institute for Informatics, Saarbr\u00fccken, Germany","institution_ids":["https://openalex.org/I4210109712"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5021368473","display_name":"Matteo Riondato","orcid":"https://orcid.org/0000-0003-2523-4420"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Matteo Riondato","raw_affiliation_strings":["Stanford University, Stanford, USA","Stanford University, Stanford , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University, Stanford, USA","institution_ids":["https://openalex.org/I97018004"]},{"raw_affiliation_string":"Stanford University, Stanford , USA","institution_ids":["https://openalex.org/I97018004"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":13,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"983","last_page":"998"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11182","display_name":"Auction Theory and Applications","score":0.9927999973297119,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10764","display_name":"Privacy-Preserving Technologies in Data","score":0.9819999933242798,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.9821445941925049},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7071189880371094},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.6285915374755859},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.609279215335846},{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.5910613536834717},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.5629301071166992},{"id":"https://openalex.org/keywords/constant","display_name":"Constant (computer programming)","score":0.5504032373428345},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.4940212368965149},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4798814356327057},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.43475258350372314},{"id":"https://openalex.org/keywords/work","display_name":"Work (physics)","score":0.41185083985328674},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.3658778667449951},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.36125361919403076},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.35071080923080444},{"id":"https://openalex.org/keywords/theoretical-computer-science","display_name":"Theoretical computer science","score":0.34516197443008423}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.9821445941925049},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7071189880371094},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.6285915374755859},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.609279215335846},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.5910613536834717},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.5629301071166992},{"id":"https://openalex.org/C2777027219","wikidata":"https://www.wikidata.org/wiki/Q1284190","display_name":"Constant (computer programming)","level":2,"score":0.5504032373428345},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.4940212368965149},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4798814356327057},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.43475258350372314},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.41185083985328674},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3658778667449951},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.36125361919403076},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.35071080923080444},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.34516197443008423},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C78519656","wikidata":"https://www.wikidata.org/wiki/Q101333","display_name":"Mechanical engineering","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/2723372.2723722","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2723372.2723722","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2015 ACM SIGMOD International Conference on Management of Data","raw_type":"proceedings-article"},{"id":"pmh:oai:iris.uniroma1.it:11573/842138","is_oa":true,"landing_page_url":"http://hdl.handle.net/11573/842138","pdf_url":"https://iris.uniroma1.it/bitstream/11573/842138/7/Anagnostopoulos_postprint_The-Importance_2015.pdf","source":{"id":"https://openalex.org/S4377196107","display_name":"IRIS Research product catalog (Sapienza University of Rome)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/conferenceObject"}],"best_oa_location":{"id":"pmh:oai:iris.uniroma1.it:11573/842138","is_oa":true,"landing_page_url":"http://hdl.handle.net/11573/842138","pdf_url":"https://iris.uniroma1.it/bitstream/11573/842138/7/Anagnostopoulos_postprint_The-Importance_2015.pdf","source":{"id":"https://openalex.org/S4377196107","display_name":"IRIS Research product catalog (Sapienza University of Rome)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/conferenceObject"},"sustainable_development_goals":[{"score":0.4399999976158142,"id":"https://metadata.un.org/sdg/8","display_name":"Decent work and economic growth"}],"awards":[{"id":"https://openalex.org/G4351865285","display_name":null,"funder_award_id":"610704, 317532","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G7036206186","display_name":"BIGDATA: Mid-Scale: DA: Analytical Approaches to Massive Data Computation with Applications to Genomics","funder_award_id":"1247581","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G7896411822","display_name":null,"funder_award_id":"IIS-1247581","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320309327","display_name":"Google","ror":"https://ror.org/00njsd438"},{"id":"https://openalex.org/F4320320300","display_name":"European Commission","ror":"https://ror.org/00k4n6c32"},{"id":"https://openalex.org/F4320321873","display_name":"Ministero dell\u2019Istruzione, dell\u2019Universit\u00e0 e della Ricerca","ror":"https://ror.org/0166hxq48"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2054593810.pdf","grobid_xml":"https://content.openalex.org/works/W2054593810.grobid-xml"},"referenced_works_count":34,"referenced_works":["https://openalex.org/W350106943","https://openalex.org/W1551839888","https://openalex.org/W1584070893","https://openalex.org/W1846955904","https://openalex.org/W1974554406","https://openalex.org/W1988695498","https://openalex.org/W1989274820","https://openalex.org/W2013922699","https://openalex.org/W2028818097","https://openalex.org/W2033442452","https://openalex.org/W2038435918","https://openalex.org/W2044712489","https://openalex.org/W2051299691","https://openalex.org/W2055733113","https://openalex.org/W2066640191","https://openalex.org/W2069396035","https://openalex.org/W2075397336","https://openalex.org/W2083293881","https://openalex.org/W2084951252","https://openalex.org/W2085891284","https://openalex.org/W2105220030","https://openalex.org/W2121117737","https://openalex.org/W2140026235","https://openalex.org/W2140890285","https://openalex.org/W2145492473","https://openalex.org/W2158880898","https://openalex.org/W2161126263","https://openalex.org/W2168144930","https://openalex.org/W2188373600","https://openalex.org/W2267033284","https://openalex.org/W2342402489","https://openalex.org/W2554839354","https://openalex.org/W2735460460","https://openalex.org/W6658850303"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W4384486036","https://openalex.org/W135177976","https://openalex.org/W1503094549","https://openalex.org/W2337920774","https://openalex.org/W4318823662","https://openalex.org/W2886410948","https://openalex.org/W2025875869","https://openalex.org/W3207526114","https://openalex.org/W4286908577"],"abstract_inverted_index":{"Crowdsourcing":[0],"is":[1,8,21,124,197,254],"a":[2,104,177,181,198,232],"computational":[3,105,146],"paradigm":[4],"whose":[5,132],"distinctive":[6,122],"feature":[7],"the":[9,18,43,57,62,79,91,94,97,125,128,151,174,202,211,246],"involvement":[10],"of":[11,17,46,70,93,100,113,120,127,153,180,183,213],"human":[12],"workers":[13,114,191,228],"in":[14,40,52,78,135,176,257],"key":[15],"steps":[16],"computation.":[19],"It":[20],"used":[22],"successfully":[23],"to":[24,33,72,149,158,166,192,201,216,231],"address":[25,82],"problems":[26],"that":[27,89,109,196,221,251],"would":[28],"be":[29],"hard":[30],"or":[31,61],"impossible":[32],"solve":[34,217],"for":[35,64,107,172],"machines.":[36],"As":[37],"we":[38,139,236],"highlight":[39],"this":[41,83,218],"work,":[42],"exclusive":[44],"use":[45,163],"nonexpert":[47],"individuals":[48],"may":[49],"prove":[50,205],"ineffective":[51],"some":[53,68],"cases,":[54],"especially":[55],"when":[56],"task":[58],"at":[59],"hand":[60],"need":[63],"accurate":[65],"solutions":[66],"demand":[67],"degree":[69],"specialization":[71],"avoid":[73],"excessive":[74],"uncertainty":[75],"and":[76,137,160,168,189,207,226,242],"inconsistency":[77],"answers.":[80],"We":[81,102,162,204],"limitation":[84],"by":[85],"proposing":[86],"an":[87,170,194],"approach":[88,253],"combines":[90],"wisdom":[92],"crowd":[95],"with":[96,115,156],"educated":[98],"opinion":[99],"experts.":[101],"present":[103],"model":[106,147,165],"crowdsourcing":[108,154,248],"envisions":[110],"two":[111],"classes":[112],"different":[116],"expertise":[117],"levels.":[118],"One":[119],"its":[121],"features":[123],"adoption":[126],"threshold":[129],"error":[130],"model,":[131],"roots":[133],"are":[134],"psychometrics":[136],"which":[138],"extend":[140],"from":[141],"previous":[142],"theoretical":[143],"work.":[144],"Our":[145],"allows":[148],"evaluate":[150,237],"performance":[152],"algorithms":[155],"respect":[157],"accuracy":[159],"cost.":[161],"our":[164,222,238,252],"develop":[167],"analyze":[169],"algorithm":[171,186,223,239],"approximating":[173],"best,":[175],"broad":[178],"sense,":[179],"set":[182],"elements.":[184],"The":[185],"uses":[187,224],"na\u00efve":[188,227],"expert":[190,225],"find":[193],"element":[195],"constant-factor":[199],"approximation":[200],"best.":[203],"upper":[206],"lower":[208],"bounds":[209],"on":[210,240],"number":[212],"comparisons":[214],"needed":[215],"problem,":[219],"showing":[220,250],"optimally":[229],"up":[230],"constant":[233],"factor.":[234],"Finally,":[235],"real":[241],"synthetic":[243],"datasets":[244],"using":[245],"CrowdFlower":[247],"platform,":[249],"also":[255],"effective":[256],"practice.":[258]},"counts_by_year":[{"year":2023,"cited_by_count":1},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":1},{"year":2019,"cited_by_count":3},{"year":2018,"cited_by_count":1},{"year":2017,"cited_by_count":1},{"year":2016,"cited_by_count":1},{"year":2015,"cited_by_count":3},{"year":2014,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2016-06-24T00:00:00"}