{"id":"https://openalex.org/W2740265435","doi":"https://doi.org/10.24963/ijcai.2017/324","title":"Improving Learning-from-Crowds through Expert Validation","display_name":"Improving Learning-from-Crowds through Expert Validation","publication_year":2017,"publication_date":"2017-07-28","ids":{"openalex":"https://openalex.org/W2740265435","doi":"https://doi.org/10.24963/ijcai.2017/324","mag":"2740265435"},"language":"en","primary_location":{"id":"doi:10.24963/ijcai.2017/324","is_oa":true,"landing_page_url":"https://doi.org/10.24963/ijcai.2017/324","pdf_url":"https://www.ijcai.org/proceedings/2017/0324.pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Twenty-Sixth International Joint Conference on Artificial Intelligence","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.ijcai.org/proceedings/2017/0324.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5107904995","display_name":"Mengchen Liu","orcid":null},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Mengchen Liu","raw_affiliation_strings":["School of Software, Tsinghua University, Beijing, P.R. China","Tsinghua National Lab for Information Science and Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Software, Tsinghua University, Beijing, P.R. China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"Tsinghua National Lab for Information Science and Technology","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101569350","display_name":"Liu Jiang","orcid":"https://orcid.org/0000-0002-7189-7762"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Liu Jiang","raw_affiliation_strings":["School of Software, Tsinghua University, Beijing, P.R. China","Tsinghua National Lab for Information Science and Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Software, Tsinghua University, Beijing, P.R. China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"Tsinghua National Lab for Information Science and Technology","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101880137","display_name":"Junlin Liu","orcid":"https://orcid.org/0000-0001-5917-2251"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Junlin Liu","raw_affiliation_strings":["School of Software, Tsinghua University, Beijing, P.R. China","Tsinghua National Lab for Information Science and Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Software, Tsinghua University, Beijing, P.R. China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"Tsinghua National Lab for Information Science and Technology","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018582673","display_name":"Xiting Wang","orcid":"https://orcid.org/0000-0002-1846-1118"},"institutions":[{"id":"https://openalex.org/I4210113369","display_name":"Microsoft Research Asia (China)","ror":"https://ror.org/0300m5276","country_code":"CN","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210113369"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiting Wang","raw_affiliation_strings":["Microsoft Research, Beijing, P.R. China","School of Software, Tsinghua University, Beijing, P.R. China","Tsinghua National Lab for Information Science and Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research, Beijing, P.R. China","institution_ids":["https://openalex.org/I4210113369"]},{"raw_affiliation_string":"School of Software, Tsinghua University, Beijing, P.R. China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"Tsinghua National Lab for Information Science and Technology","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100606995","display_name":"Jun Zhu","orcid":"https://orcid.org/0000-0002-6254-2388"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jun Zhu","raw_affiliation_strings":["Department of Computer Science and Technology, Tsinghua University, Beijing, P.R. China","Tsinghua National Lab for Information Science and Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Technology, Tsinghua University, Beijing, P.R. China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"Tsinghua National Lab for Information Science and Technology","institution_ids":[]}]},{"author_position":"last","author":{"id":null,"display_name":"Shixia Liu","orcid":null},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shixia Liu","raw_affiliation_strings":["School of Software, Tsinghua University, Beijing, P.R. China","Tsinghua National Lab for Information Science and Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Software, Tsinghua University, Beijing, P.R. China","institution_ids":["https://openalex.org/I99065089"]},{"raw_affiliation_string":"Tsinghua National Lab for Information Science and Technology","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":28,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"2329","last_page":"2336"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11512","display_name":"Anomaly Detection Techniques and Applications","score":0.9950000047683716,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9940999746322632,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.764934778213501},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.755508542060852},{"id":"https://openalex.org/keywords/crowds","display_name":"Crowds","score":0.722727358341217},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.7180110216140747},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.6121923923492432},{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.5949939489364624},{"id":"https://openalex.org/keywords/selection","display_name":"Selection (genetic algorithm)","score":0.45692750811576843},{"id":"https://openalex.org/keywords/bayesian-inference","display_name":"Bayesian inference","score":0.44425371289253235},{"id":"https://openalex.org/keywords/expert-system","display_name":"Expert system","score":0.4411911964416504},{"id":"https://openalex.org/keywords/bayesian-probability","display_name":"Bayesian probability","score":0.4304811954498291}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.764934778213501},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.755508542060852},{"id":"https://openalex.org/C2777852691","wikidata":"https://www.wikidata.org/wiki/Q13430821","display_name":"Crowds","level":2,"score":0.722727358341217},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.7180110216140747},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.6121923923492432},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.5949939489364624},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.45692750811576843},{"id":"https://openalex.org/C160234255","wikidata":"https://www.wikidata.org/wiki/Q812535","display_name":"Bayesian inference","level":3,"score":0.44425371289253235},{"id":"https://openalex.org/C58328972","wikidata":"https://www.wikidata.org/wiki/Q184609","display_name":"Expert system","level":2,"score":0.4411911964416504},{"id":"https://openalex.org/C107673813","wikidata":"https://www.wikidata.org/wiki/Q812534","display_name":"Bayesian probability","level":2,"score":0.4304811954498291},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.24963/ijcai.2017/324","is_oa":true,"landing_page_url":"https://doi.org/10.24963/ijcai.2017/324","pdf_url":"https://www.ijcai.org/proceedings/2017/0324.pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Twenty-Sixth International Joint Conference on Artificial Intelligence","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.24963/ijcai.2017/324","is_oa":true,"landing_page_url":"https://doi.org/10.24963/ijcai.2017/324","pdf_url":"https://www.ijcai.org/proceedings/2017/0324.pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Twenty-Sixth International Joint Conference on Artificial Intelligence","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1919465925","display_name":null,"funder_award_id":"61621136008","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G368795326","display_name":null,"funder_award_id":"61620106010","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5207934879","display_name":"\u673a\u5668\u5b66\u4e60\u4e2d\u7684\u53c2\u6570\u8c03\u6574\u53ef\u89c6\u5206\u6790\u7814\u7a76","funder_award_id":"61672308","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5606024398","display_name":null,"funder_award_id":"61672308","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2740265435.pdf","grobid_xml":"https://content.openalex.org/works/W2740265435.grobid-xml"},"referenced_works_count":32,"referenced_works":["https://openalex.org/W9014458","https://openalex.org/W1493526108","https://openalex.org/W1501025848","https://openalex.org/W1686810756","https://openalex.org/W1901365844","https://openalex.org/W1970381522","https://openalex.org/W1995875735","https://openalex.org/W2041404167","https://openalex.org/W2074275752","https://openalex.org/W2105268242","https://openalex.org/W2108598243","https://openalex.org/W2115394472","https://openalex.org/W2128475742","https://openalex.org/W2129345386","https://openalex.org/W2140890285","https://openalex.org/W2151023586","https://openalex.org/W2152009989","https://openalex.org/W2157791002","https://openalex.org/W2187444452","https://openalex.org/W2273171220","https://openalex.org/W2475282416","https://openalex.org/W2549690445","https://openalex.org/W2571859924","https://openalex.org/W2578209164","https://openalex.org/W2885321846","https://openalex.org/W2903158431","https://openalex.org/W2963641944","https://openalex.org/W2978725006","https://openalex.org/W3004533406","https://openalex.org/W3023540311","https://openalex.org/W4230030242","https://openalex.org/W4239072543"],"related_works":["https://openalex.org/W2407375987","https://openalex.org/W3049691116","https://openalex.org/W2505726097","https://openalex.org/W2010643158","https://openalex.org/W2106867672","https://openalex.org/W3081214562","https://openalex.org/W4310268968","https://openalex.org/W2950975704","https://openalex.org/W4297815943","https://openalex.org/W2053745677"],"abstract_inverted_index":{"Although":[0],"several":[1],"effective":[2],"learning-from-crowd":[3],"methods":[4],"have":[5,50],"been":[6],"developed":[7,51],"to":[8,56,71,101],"infer":[9],"correct":[10],"labels":[11,67],"from":[12,99],"noisy":[13],"crowdsourced":[14],"labels,":[15],"a":[16,28,52,88],"method":[17,95],"for":[18],"post-processed":[19],"expert":[20,46,66,97],"validation":[21],"is":[22,33],"still":[23],"needed.":[24],"This":[25],"paper":[26],"introduces":[27],"semi-supervised":[29],"learning":[30],"algorithm":[31],"that":[32,86],"capable":[34],"of":[35,45,60],"selecting":[36],"the":[37,43,58,61,105],"most":[38,62],"informative":[39,63],"instances":[40,73],"and":[41,82],"maximizing":[42],"influence":[44],"labels.":[47],"Specifically,":[48],"we":[49],"complete":[53],"uncertainty":[54],"assessment":[55],"facilitate":[57],"selection":[59],"instances.":[64],"The":[65],"are":[68],"then":[69],"propagated":[70],"similar":[72],"via":[74],"regularized":[75],"Bayesian":[76],"inference.":[77],"Experiments":[78],"on":[79],"both":[80],"real-world":[81],"simulated":[83],"datasets":[84],"indicate":[85],"given":[87],"specific":[89],"accuracy":[90],"goal":[91],"(e.g.,":[92],"95%)":[93],"our":[94],"reduces":[96],"effort":[98],"39%":[100],"60%":[102],"compared":[103],"with":[104],"state-of-the-art":[106],"method.":[107]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2023,"cited_by_count":3},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":2},{"year":2020,"cited_by_count":5},{"year":2019,"cited_by_count":5},{"year":2018,"cited_by_count":7},{"year":2017,"cited_by_count":2}],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2025-10-10T00:00:00"}