{"id":"https://openalex.org/W2018369380","doi":"https://doi.org/10.1145/2806416.2806627","title":"Improving Label Quality in Crowdsourcing Using Noise Correction","display_name":"Improving Label Quality in Crowdsourcing Using Noise Correction","publication_year":2015,"publication_date":"2015-10-17","ids":{"openalex":"https://openalex.org/W2018369380","doi":"https://doi.org/10.1145/2806416.2806627","mag":"2018369380"},"language":"en","primary_location":{"id":"doi:10.1145/2806416.2806627","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2806416.2806627","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 24th ACM International on Conference on Information and Knowledge Management","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101994917","display_name":"Jing Zhang","orcid":"https://orcid.org/0000-0003-2494-8077"},"institutions":[{"id":"https://openalex.org/I36399199","display_name":"Nanjing University of Science and Technology","ror":"https://ror.org/00xp9wg62","country_code":"CN","type":"education","lineage":["https://openalex.org/I36399199"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Jing Zhang","raw_affiliation_strings":["Nanjing University of Science and Technology, Nanjing, China"],"affiliations":[{"raw_affiliation_string":"Nanjing University of Science and Technology, Nanjing, China","institution_ids":["https://openalex.org/I36399199"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051706630","display_name":"Victor S. Sheng","orcid":"https://orcid.org/0000-0003-4960-174X"},"institutions":[{"id":"https://openalex.org/I32038505","display_name":"University of Central Arkansas","ror":"https://ror.org/029bp0k25","country_code":"US","type":"education","lineage":["https://openalex.org/I32038505"]},{"id":"https://openalex.org/I4210152127","display_name":"Conway School of Landscape Design","ror":"https://ror.org/04q7y8a54","country_code":"US","type":"education","lineage":["https://openalex.org/I4210152127"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Victor S. Sheng","raw_affiliation_strings":["University of Central Arkansas, Conway, AR, USA"],"affiliations":[{"raw_affiliation_string":"University of Central Arkansas, Conway, AR, USA","institution_ids":["https://openalex.org/I32038505","https://openalex.org/I4210152127"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101774308","display_name":"Jian Wu","orcid":"https://orcid.org/0000-0002-8759-7107"},"institutions":[{"id":"https://openalex.org/I3923682","display_name":"Soochow University","ror":"https://ror.org/05t8y2r12","country_code":"CN","type":"education","lineage":["https://openalex.org/I3923682"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jian Wu","raw_affiliation_strings":["Soochow University, Suzhou, China"],"affiliations":[{"raw_affiliation_string":"Soochow University, Suzhou, China","institution_ids":["https://openalex.org/I3923682"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050437336","display_name":"Xiaoqin Fu","orcid":"https://orcid.org/0000-0001-6354-8960"},"institutions":[{"id":"https://openalex.org/I32038505","display_name":"University of Central Arkansas","ror":"https://ror.org/029bp0k25","country_code":"US","type":"education","lineage":["https://openalex.org/I32038505"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xiaoqin Fu","raw_affiliation_strings":["University of Central Arkansas, University of Central Arkansas, AR, USA"],"affiliations":[{"raw_affiliation_string":"University of Central Arkansas, University of Central Arkansas, AR, USA","institution_ids":["https://openalex.org/I32038505"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5080738591","display_name":"Xindong Wu","orcid":"https://orcid.org/0000-0003-2396-1704"},"institutions":[{"id":"https://openalex.org/I16365422","display_name":"Hefei University of Technology","ror":"https://ror.org/02czkny70","country_code":"CN","type":"education","lineage":["https://openalex.org/I16365422"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xindong Wu","raw_affiliation_strings":["Hefei University of Technology, Hefei, China"],"affiliations":[{"raw_affiliation_string":"Hefei University of Technology, Hefei, China","institution_ids":["https://openalex.org/I16365422"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5101994917"],"corresponding_institution_ids":["https://openalex.org/I36399199"],"apc_list":null,"apc_paid":null,"fwci":6.0335,"has_fulltext":false,"cited_by_count":18,"citation_normalized_percentile":{"value":0.95789821,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"1931","last_page":"1934"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9966999888420105,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11606","display_name":"Infrastructure Maintenance and Monitoring","score":0.9868000149726868,"subfield":{"id":"https://openalex.org/subfields/2205","display_name":"Civil and Structural Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.9376474022865295},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.8339104652404785},{"id":"https://openalex.org/keywords/majority-rule","display_name":"Majority rule","score":0.7679792046546936},{"id":"https://openalex.org/keywords/noise","display_name":"Noise (video)","score":0.7550631761550903},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7509772777557373},{"id":"https://openalex.org/keywords/ground-truth","display_name":"Ground truth","score":0.7201563119888306},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.6082950234413147},{"id":"https://openalex.org/keywords/baseline","display_name":"Baseline (sea)","score":0.6009560823440552},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5267300605773926},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4995994567871094},{"id":"https://openalex.org/keywords/noise-measurement","display_name":"Noise measurement","score":0.48160311579704285},{"id":"https://openalex.org/keywords/voting","display_name":"Voting","score":0.4534388780593872},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.4262772798538208},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.3235485255718231},{"id":"https://openalex.org/keywords/noise-reduction","display_name":"Noise reduction","score":0.22486358880996704}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.9376474022865295},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.8339104652404785},{"id":"https://openalex.org/C153668964","wikidata":"https://www.wikidata.org/wiki/Q27636","display_name":"Majority rule","level":2,"score":0.7679792046546936},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.7550631761550903},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7509772777557373},{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.7201563119888306},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.6082950234413147},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.6009560823440552},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5267300605773926},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4995994567871094},{"id":"https://openalex.org/C29265498","wikidata":"https://www.wikidata.org/wiki/Q7047719","display_name":"Noise measurement","level":3,"score":0.48160311579704285},{"id":"https://openalex.org/C520049643","wikidata":"https://www.wikidata.org/wiki/Q189760","display_name":"Voting","level":3,"score":0.4534388780593872},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4262772798538208},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3235485255718231},{"id":"https://openalex.org/C163294075","wikidata":"https://www.wikidata.org/wiki/Q581861","display_name":"Noise reduction","level":2,"score":0.22486358880996704},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.0},{"id":"https://openalex.org/C127313418","wikidata":"https://www.wikidata.org/wiki/Q1069","display_name":"Geology","level":0,"score":0.0},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0},{"id":"https://openalex.org/C111368507","wikidata":"https://www.wikidata.org/wiki/Q43518","display_name":"Oceanography","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/2806416.2806627","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2806416.2806627","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 24th ACM International on Conference on Information and Knowledge Management","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.5799999833106995,"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":12,"referenced_works":["https://openalex.org/W41502859","https://openalex.org/W1562615108","https://openalex.org/W1818395637","https://openalex.org/W2027977399","https://openalex.org/W2098370488","https://openalex.org/W2113878109","https://openalex.org/W2125943921","https://openalex.org/W2134305421","https://openalex.org/W2142518823","https://openalex.org/W2167460663","https://openalex.org/W2394968026","https://openalex.org/W3100570787"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W4384486036","https://openalex.org/W135177976","https://openalex.org/W1503094549","https://openalex.org/W2337920774","https://openalex.org/W2886410948","https://openalex.org/W2025875869","https://openalex.org/W2039876276","https://openalex.org/W2605569989","https://openalex.org/W2295086410"],"abstract_inverted_index":{"This":[0],"paper":[1],"proposes":[2],"a":[3,84,107],"novel":[4],"framework":[5,67],"that":[6,64,80],"introduces":[7],"noise":[8,28,109],"correction":[9,29,110],"techniques":[10],"to":[11,34],"further":[12],"improve":[13,69],"label":[14,70],"quality":[15,71],"after":[16],"ground":[17,53],"truth":[18,54],"inference":[19,74],"in":[20],"crowdsourcing.":[21],"In":[22],"the":[23,38,43,52,66,78,91,96,100],"framework,":[24],"an":[25],"adaptive":[26],"voting":[27],"algorithm":[30,92],"(AVNC)":[31],"is":[32],"proposed":[33],"identify":[35],"and":[36,88,99],"correct":[37],"most":[39],"likely":[40],"noises":[41],"with":[42],"help":[44],"of":[45,48,73,98,102],"estimated":[46],"qualities":[47],"labelers":[49],"provided":[50],"by":[51],"inference.":[55],"The":[56],"experimental":[57],"results":[58],"on":[59],"two":[60],"real-world":[61],"datasets":[62],"show":[63],"(1)":[65],"can":[68],"regardless":[72],"algorithms,":[75],"especially":[76],"under":[77],"circumstance":[79],"each":[81],"example":[82],"has":[83],"few":[85],"noisy":[86],"labels;":[87],"(2)":[89],"since":[90],"AVNC":[93],"considers":[94],"both":[95],"number":[97],"probability":[101],"potential":[103],"noises,":[104],"it":[105],"outperforms":[106],"baseline":[108],"algorithm.":[111]},"counts_by_year":[{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":1},{"year":2019,"cited_by_count":1},{"year":2018,"cited_by_count":2},{"year":2017,"cited_by_count":6},{"year":2016,"cited_by_count":4}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}