{"id":"https://openalex.org/W2074275752","doi":"https://doi.org/10.1145/2723372.2723731","title":"Minimizing Efforts in Validating Crowd Answers","display_name":"Minimizing Efforts in Validating Crowd Answers","publication_year":2015,"publication_date":"2015-05-27","ids":{"openalex":"https://openalex.org/W2074275752","doi":"https://doi.org/10.1145/2723372.2723731","mag":"2074275752"},"language":"en","primary_location":{"id":"doi:10.1145/2723372.2723731","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2723372.2723731","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2015 ACM SIGMOD International Conference on Management of Data","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"http://infoscience.epfl.ch/record/204725","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5051219382","display_name":"Quoc Viet Hung Nguyen","orcid":"https://orcid.org/0000-0002-9687-1315"},"institutions":[{"id":"https://openalex.org/I5124864","display_name":"\u00c9cole Polytechnique F\u00e9d\u00e9rale de Lausanne","ror":"https://ror.org/02s376052","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I5124864"]}],"countries":["CH"],"is_corresponding":true,"raw_author_name":"Nguyen Quoc Viet Hung","raw_affiliation_strings":["EPFL, Lausanne, Switzerland"],"affiliations":[{"raw_affiliation_string":"EPFL, Lausanne, Switzerland","institution_ids":["https://openalex.org/I5124864"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028221774","display_name":"Chi Thang Duong","orcid":"https://orcid.org/0000-0002-2504-8719"},"institutions":[{"id":"https://openalex.org/I5124864","display_name":"\u00c9cole Polytechnique F\u00e9d\u00e9rale de Lausanne","ror":"https://ror.org/02s376052","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I5124864"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Duong Chi Thang","raw_affiliation_strings":["EPFL, Lausanne, Switzerland"],"affiliations":[{"raw_affiliation_string":"EPFL, Lausanne, Switzerland","institution_ids":["https://openalex.org/I5124864"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070529144","display_name":"Matthias Weidlich","orcid":"https://orcid.org/0000-0003-3325-7227"},"institutions":[{"id":"https://openalex.org/I47508984","display_name":"Imperial College London","ror":"https://ror.org/041kmwe10","country_code":"GB","type":"education","lineage":["https://openalex.org/I47508984"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Matthias Weidlich","raw_affiliation_strings":["Imperial College London, London, United Kingdom"],"affiliations":[{"raw_affiliation_string":"Imperial College London, London, United Kingdom","institution_ids":["https://openalex.org/I47508984"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5001443156","display_name":"Karl Aberer","orcid":"https://orcid.org/0000-0003-3005-7342"},"institutions":[{"id":"https://openalex.org/I5124864","display_name":"\u00c9cole Polytechnique F\u00e9d\u00e9rale de Lausanne","ror":"https://ror.org/02s376052","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I5124864"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Karl Aberer","raw_affiliation_strings":["EPFL, Lausanne, Switzerland"],"affiliations":[{"raw_affiliation_string":"EPFL, Lausanne, Switzerland","institution_ids":["https://openalex.org/I5124864"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5051219382"],"corresponding_institution_ids":["https://openalex.org/I5124864"],"apc_list":null,"apc_paid":null,"fwci":24.6381,"has_fulltext":false,"cited_by_count":82,"citation_normalized_percentile":{"value":0.99362462,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"999","last_page":"1014"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9926000237464905,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.9822999835014343,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.9655275344848633},{"id":"https://openalex.org/keywords/correctness","display_name":"Correctness","score":0.9401164054870605},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7802322506904602},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.682030439376831},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.6765710711479187},{"id":"https://openalex.org/keywords/probabilistic-logic","display_name":"Probabilistic logic","score":0.5480991005897522},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.5081005096435547},{"id":"https://openalex.org/keywords/range","display_name":"Range (aeronautics)","score":0.446611613035202},{"id":"https://openalex.org/keywords/baseline","display_name":"Baseline (sea)","score":0.43780073523521423},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.3690266013145447},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3607938289642334},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3204934597015381},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.23375949263572693},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.11218556761741638},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.08889514207839966}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.9655275344848633},{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.9401164054870605},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7802322506904602},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.682030439376831},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.6765710711479187},{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.5480991005897522},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.5081005096435547},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.446611613035202},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.43780073523521423},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3690266013145447},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3607938289642334},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3204934597015381},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.23375949263572693},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.11218556761741638},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.08889514207839966},{"id":"https://openalex.org/C127313418","wikidata":"https://www.wikidata.org/wiki/Q1069","display_name":"Geology","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C146978453","wikidata":"https://www.wikidata.org/wiki/Q3798668","display_name":"Aerospace engineering","level":1,"score":0.0},{"id":"https://openalex.org/C111368507","wikidata":"https://www.wikidata.org/wiki/Q43518","display_name":"Oceanography","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1145/2723372.2723731","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2723372.2723731","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2015 ACM SIGMOD International Conference on Management of Data","raw_type":"proceedings-article"},{"id":"pmh:oai:infoscience.tind.io:204725","is_oa":true,"landing_page_url":"http://infoscience.epfl.ch/record/204725","pdf_url":null,"source":{"id":"https://openalex.org/S4306400487","display_name":"Infoscience (Ecole Polytechnique F\u00e9d\u00e9rale de Lausanne)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"conference proceedings"},{"id":"pmh:oai:research-repository.griffith.edu.au:10072/348004","is_oa":false,"landing_page_url":"http://hdl.handle.net/10072/348004","pdf_url":null,"source":{"id":"https://openalex.org/S4306402548","display_name":"Griffith Research Online (Griffith University, Queensland, Australia)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I11701301","host_organization_name":"Griffith University","host_organization_lineage":["https://openalex.org/I11701301"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Conference output"},{"id":"pmh:oai:spiral.imperial.ac.uk:10044/1/32558","is_oa":false,"landing_page_url":"http://hdl.handle.net/10044/1/32558","pdf_url":null,"source":{"id":"https://openalex.org/S4306401396","display_name":"Spiral (Imperial College London)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I47508984","host_organization_name":"Imperial College London","host_organization_lineage":["https://openalex.org/I47508984"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"2015 ACM SIGMOD International Conference on Management of Data","raw_type":"Conference Paper"}],"best_oa_location":{"id":"pmh:oai:infoscience.tind.io:204725","is_oa":true,"landing_page_url":"http://infoscience.epfl.ch/record/204725","pdf_url":null,"source":{"id":"https://openalex.org/S4306400487","display_name":"Infoscience (Ecole Polytechnique F\u00e9d\u00e9rale de Lausanne)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"conference proceedings"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":54,"referenced_works":["https://openalex.org/W9014458","https://openalex.org/W96765277","https://openalex.org/W166707811","https://openalex.org/W1501025848","https://openalex.org/W1501049730","https://openalex.org/W1605255363","https://openalex.org/W1639032689","https://openalex.org/W1713409046","https://openalex.org/W2000855935","https://openalex.org/W2003684739","https://openalex.org/W2004026774","https://openalex.org/W2036431079","https://openalex.org/W2041404167","https://openalex.org/W2063103859","https://openalex.org/W2069897123","https://openalex.org/W2072651985","https://openalex.org/W2076357412","https://openalex.org/W2092262612","https://openalex.org/W2095779006","https://openalex.org/W2098642746","https://openalex.org/W2098865355","https://openalex.org/W2104814277","https://openalex.org/W2106476394","https://openalex.org/W2107966677","https://openalex.org/W2110411158","https://openalex.org/W2113878109","https://openalex.org/W2114269021","https://openalex.org/W2116664070","https://openalex.org/W2119320829","https://openalex.org/W2121117737","https://openalex.org/W2122410182","https://openalex.org/W2123885506","https://openalex.org/W2125273615","https://openalex.org/W2125759315","https://openalex.org/W2138965424","https://openalex.org/W2140890285","https://openalex.org/W2152150600","https://openalex.org/W2155189155","https://openalex.org/W2155756297","https://openalex.org/W2159296364","https://openalex.org/W2164124780","https://openalex.org/W2169585110","https://openalex.org/W2170989440","https://openalex.org/W2296044363","https://openalex.org/W2401447850","https://openalex.org/W2611973430","https://openalex.org/W2976265969","https://openalex.org/W2978725006","https://openalex.org/W4205167958","https://openalex.org/W4210651582","https://openalex.org/W6603840481","https://openalex.org/W6637487752","https://openalex.org/W6678087030","https://openalex.org/W6768433217"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W4384486036","https://openalex.org/W135177976","https://openalex.org/W1503094549","https://openalex.org/W2337920774","https://openalex.org/W4318823662","https://openalex.org/W2886410948","https://openalex.org/W2025875869","https://openalex.org/W3207526114","https://openalex.org/W4286908577"],"abstract_inverted_index":{"In":[0,172],"recent":[1],"years,":[2],"crowdsourcing":[3,20],"has":[4,187],"become":[5],"essential":[6],"in":[7,54,93],"a":[8,51,81,125],"wide":[9],"range":[10],"of":[11,15,19,24,32,95,98,103,127,159,193],"Web":[12],"applications.":[13],"One":[14],"the":[16,22,35,88,112,119,133,138,194],"biggest":[17],"challenges":[18],"is":[21,60,64,71],"quality":[23,46,129],"crowd":[25,56],"answers":[26,57,130],"as":[27],"workers":[28],"have":[29,48],"wide-ranging":[30],"levels":[31],"expertise":[33],"and":[34,73,101,148],"worker":[36],"community":[37],"may":[38],"contain":[39],"faulty":[40,104],"workers.":[41,105],"Although":[42],"various":[43],"techniques":[44,154],"for":[45,168,175,190],"control":[47],"been":[49,188],"proposed,":[50],"post-processing":[52],"phase":[53],"which":[55],"are":[58],"validated":[59],"still":[61],"required.":[62],"Validation":[63],"typically":[65],"conducted":[66],"by":[67,115],"experts,":[68],"whose":[69],"availability":[70],"limited":[72],"who":[74],"incur":[75],"high":[76,128],"costs.":[77],"Therefore,":[78],"we":[79,178],"develop":[80],"probabilistic":[82],"model":[83],"that":[84,152],"helps":[85],"to":[86,110,157,163,181],"identify":[87],"most":[89,120,176],"beneficial":[90],"validation":[91],"questions":[92],"terms":[94],"both,":[96],"improvement":[97],"result":[99,170],"correctness":[100,183],"detection":[102],"Our":[106,142],"approach":[107],"allows":[108],"us":[109],"guide":[111],"expert's":[113],"work":[114],"collecting":[116],"input":[117,186],"on":[118],"problematic":[121],"cases,":[122,177],"thereby":[123],"achieving":[124],"set":[126],"even":[131],"if":[132],"expert":[134,160,185],"does":[135],"not":[136],"validate":[137],"complete":[139],"answer":[140],"set.":[141],"comprehensive":[143],"evaluation":[144],"using":[145],"both":[146],"real-world":[147],"synthetic":[149],"datasets":[150],"demonstrates":[151],"our":[153],"save":[155],"up":[156],"50%":[158],"efforts":[161],"compared":[162],"baseline":[164],"methods":[165],"when":[166],"striving":[167],"perfect":[169,182],"correctness.":[171],"absolute":[173],"terms,":[174],"achieve":[179],"close":[180],"after":[184],"sought":[189],"only":[191],"20\\%":[192],"questions.":[195]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":8},{"year":2020,"cited_by_count":7},{"year":2019,"cited_by_count":11},{"year":2018,"cited_by_count":18},{"year":2017,"cited_by_count":16},{"year":2016,"cited_by_count":8},{"year":2015,"cited_by_count":7}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}