{"id":"https://openalex.org/W4401567640","doi":"https://doi.org/10.1109/tnnls.2024.3438680","title":"Label Consistency-Based Ground Truth Inference for Crowdsourcing","display_name":"Label Consistency-Based Ground Truth Inference for Crowdsourcing","publication_year":2024,"publication_date":"2024-08-14","ids":{"openalex":"https://openalex.org/W4401567640","doi":"https://doi.org/10.1109/tnnls.2024.3438680","pmid":"https://pubmed.ncbi.nlm.nih.gov/39141458"},"language":"en","primary_location":{"id":"doi:10.1109/tnnls.2024.3438680","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tnnls.2024.3438680","pdf_url":null,"source":{"id":"https://openalex.org/S4210175523","display_name":"IEEE Transactions on Neural Networks and Learning Systems","issn_l":"2162-237X","issn":["2162-237X","2162-2388"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Neural Networks and Learning Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100568705","display_name":"Jiao Li","orcid":"https://orcid.org/0009-0002-0809-8287"},"institutions":[{"id":"https://openalex.org/I3124059619","display_name":"China University of Geosciences","ror":"https://ror.org/04gcegc37","country_code":"CN","type":"education","lineage":["https://openalex.org/I3124059619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiao Li","raw_affiliation_strings":["School of Computer Science, China University of Geosciences, Wuhan, China"],"raw_orcid":"https://orcid.org/0009-0002-0809-8287","affiliations":[{"raw_affiliation_string":"School of Computer Science, China University of Geosciences, Wuhan, China","institution_ids":["https://openalex.org/I3124059619"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045568216","display_name":"Liangxiao Jiang","orcid":"https://orcid.org/0000-0003-2201-3526"},"institutions":[{"id":"https://openalex.org/I3124059619","display_name":"China University of Geosciences","ror":"https://ror.org/04gcegc37","country_code":"CN","type":"education","lineage":["https://openalex.org/I3124059619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Liangxiao Jiang","raw_affiliation_strings":["School of Computer Science, China University of Geosciences, Wuhan, China"],"raw_orcid":"https://orcid.org/0000-0003-2201-3526","affiliations":[{"raw_affiliation_string":"School of Computer Science, China University of Geosciences, Wuhan, China","institution_ids":["https://openalex.org/I3124059619"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5103286642","display_name":"Wenjun Zhang","orcid":"https://orcid.org/0000-0002-7269-0376"},"institutions":[{"id":"https://openalex.org/I3124059619","display_name":"China University of Geosciences","ror":"https://ror.org/04gcegc37","country_code":"CN","type":"education","lineage":["https://openalex.org/I3124059619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wenjun Zhang","raw_affiliation_strings":["School of Computer Science, China University of Geosciences, Wuhan, China"],"raw_orcid":"https://orcid.org/0000-0002-7269-0376","affiliations":[{"raw_affiliation_string":"School of Computer Science, China University of Geosciences, Wuhan, China","institution_ids":["https://openalex.org/I3124059619"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I3124059619"],"apc_list":null,"apc_paid":null,"fwci":2.5566,"has_fulltext":false,"cited_by_count":8,"citation_normalized_percentile":{"value":0.91835616,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":99},"biblio":{"volume":"36","issue":"5","first_page":"9408","last_page":"9421"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9660000205039978,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11652","display_name":"Imbalanced Data Classification Techniques","score":0.9603999853134155,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.8811167478561401},{"id":"https://openalex.org/keywords/ground-truth","display_name":"Ground truth","score":0.859229564666748},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.7488549947738647},{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.6565991640090942},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.46420297026634216},{"id":"https://openalex.org/keywords/common-ground","display_name":"Common ground","score":0.448752760887146},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.417726069688797},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.33876746892929077},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.15406113862991333},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.09006962180137634},{"id":"https://openalex.org/keywords/social-psychology","display_name":"Social psychology","score":0.0857243537902832}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.8811167478561401},{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.859229564666748},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.7488549947738647},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.6565991640090942},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.46420297026634216},{"id":"https://openalex.org/C2777877512","wikidata":"https://www.wikidata.org/wiki/Q1116097","display_name":"Common ground","level":2,"score":0.448752760887146},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.417726069688797},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.33876746892929077},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.15406113862991333},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.09006962180137634},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.0857243537902832}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tnnls.2024.3438680","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tnnls.2024.3438680","pdf_url":null,"source":{"id":"https://openalex.org/S4210175523","display_name":"IEEE Transactions on Neural Networks and Learning Systems","issn_l":"2162-237X","issn":["2162-237X","2162-2388"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Neural Networks and Learning Systems","raw_type":"journal-article"},{"id":"pmid:39141458","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/39141458","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on neural networks and learning systems","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G4253881775","display_name":null,"funder_award_id":"62276241","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":43,"referenced_works":["https://openalex.org/W9014458","https://openalex.org/W2034771068","https://openalex.org/W2063606769","https://openalex.org/W2113878109","https://openalex.org/W2125943921","https://openalex.org/W2141708418","https://openalex.org/W2163522723","https://openalex.org/W2163952039","https://openalex.org/W2295086410","https://openalex.org/W2410545331","https://openalex.org/W2582409643","https://openalex.org/W2807400035","https://openalex.org/W2883462823","https://openalex.org/W2893310213","https://openalex.org/W2937746382","https://openalex.org/W2964397271","https://openalex.org/W2970199969","https://openalex.org/W2991697946","https://openalex.org/W2997546679","https://openalex.org/W2998826047","https://openalex.org/W3025446445","https://openalex.org/W3034786938","https://openalex.org/W3084298792","https://openalex.org/W3114228236","https://openalex.org/W3164970261","https://openalex.org/W3204943967","https://openalex.org/W3208783258","https://openalex.org/W3217201372","https://openalex.org/W4224995877","https://openalex.org/W4281256703","https://openalex.org/W4283803055","https://openalex.org/W4284878793","https://openalex.org/W4293409613","https://openalex.org/W4295022859","https://openalex.org/W4312162366","https://openalex.org/W4323925706","https://openalex.org/W4389252535","https://openalex.org/W4391648979","https://openalex.org/W6634275384","https://openalex.org/W6636859864","https://openalex.org/W6679959949","https://openalex.org/W6680957539","https://openalex.org/W6695650742"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W135177976","https://openalex.org/W4384486036","https://openalex.org/W1503094549","https://openalex.org/W2337920774","https://openalex.org/W4286908577","https://openalex.org/W2886410948","https://openalex.org/W2025875869","https://openalex.org/W4307535225","https://openalex.org/W2905544777"],"abstract_inverted_index":{"In":[0,57,97],"crowdsourcing":[1],"scenarios,":[2],"we":[3,60,87,99,131,147,160],"can":[4],"obtain":[5],"each":[6,171],"instance's":[7],"multiple":[8,45],"noisy":[9,46],"labels":[10,47],"from":[11],"different":[12,137],"crowd":[13],"workers":[14,103,111,138],"and":[15,117,177,194],"then":[16],"infer":[17],"its":[18],"unknown":[19],"ground":[20,24,37,55,82,92,209],"truth":[21,25,38,93,210],"via":[22],"a":[23,49,64,89,106,118],"inference":[26,39,94,211],"method.":[27,96],"However,":[28],"to":[29,43,62,165,185],"the":[30,35,54,73,76,81,114,129,133,140,145,149,153,167,174,204],"best":[31],"of":[32,75,136,152,170,182],"our":[33],"knowledge,":[34],"existing":[36],"methods":[40],"always":[41],"attempt":[42],"aggregate":[44],"into":[48],"single":[50],"consensus":[51],"label":[52,68,74,90,134,150,179,183,207],"as":[53,80],"truth.":[56,83],"this":[58,85],"article,":[59],"aim":[61],"explore":[63],"new":[65],"strategy,":[66],"i.e.,":[67],"selection,":[69],"which":[70],"directly":[71],"selects":[72],"highest":[77],"quality":[78,169],"worker":[79,155,172],"To":[84,127,143],"end,":[86],"propose":[88],"consistency-based":[91],"(LCGTI)":[95],"LCGTI,":[98],"argue":[100],"that":[101,199],"high-quality":[102],"should":[104],"have":[105],"low":[107,119],"bias":[108],"with":[109,121],"other":[110,205],"in":[112,123],"labeling":[113,124,168],"same":[115,141,154],"instances":[116],"variance":[120],"themselves":[122],"similar":[125,157],"instances.":[126,142,158],"estimate":[128,144],"bias,":[130],"calculate":[132,148,166],"consistency":[135,151],"on":[139,156,173,191],"variance,":[146],"Finally,":[159],"combine":[161],"these":[162],"two":[163,195],"components":[164],"inferred":[175],"instance":[176],"perform":[178],"selection":[180],"instead":[181],"aggregation":[184],"achieve":[186],"inference.":[187],"The":[188],"experimental":[189],"results":[190],"34":[192],"simulated":[193],"real-world":[196],"datasets":[197],"show":[198],"LCGTI":[200],"significantly":[201],"outperforms":[202],"all":[203],"state-of-the-art":[206],"aggregation-based":[208],"methods.":[212]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":6}],"updated_date":"2025-12-26T23:08:49.675405","created_date":"2025-10-10T00:00:00"}