{"id":"https://openalex.org/W4390621563","doi":"https://doi.org/10.1109/fie58773.2023.10343508","title":"Quality Control of Crowd Labeling for Improving the Quality of Peer Assessments","display_name":"Quality Control of Crowd Labeling for Improving the Quality of Peer Assessments","publication_year":2023,"publication_date":"2023-10-18","ids":{"openalex":"https://openalex.org/W4390621563","doi":"https://doi.org/10.1109/fie58773.2023.10343508"},"language":"en","primary_location":{"id":"doi:10.1109/fie58773.2023.10343508","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/fie58773.2023.10343508","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE Frontiers in Education Conference (FIE)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5114116037","display_name":"Banpreet Singh Chhabra","orcid":null},"institutions":[{"id":"https://openalex.org/I137902535","display_name":"North Carolina State University","ror":"https://ror.org/04tj63d06","country_code":"US","type":"education","lineage":["https://openalex.org/I137902535"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Banpreet Singh Chhabra","raw_affiliation_strings":["North Carolina State University,Department of Computer Science,Raleigh,NC,US","Department of Computer Science, North Carolina State University, Raleigh, NC, US"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"North Carolina State University,Department of Computer Science,Raleigh,NC,US","institution_ids":["https://openalex.org/I137902535"]},{"raw_affiliation_string":"Department of Computer Science, North Carolina State University, Raleigh, NC, US","institution_ids":["https://openalex.org/I137902535"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5040537321","display_name":"Edward F. Gehringer","orcid":"https://orcid.org/0000-0002-5217-2643"},"institutions":[{"id":"https://openalex.org/I137902535","display_name":"North Carolina State University","ror":"https://ror.org/04tj63d06","country_code":"US","type":"education","lineage":["https://openalex.org/I137902535"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Edward F. Gehringer","raw_affiliation_strings":["North Carolina State University,Department of Computer Science,Raleigh,NC,US","Department of Computer Science, North Carolina State University, Raleigh, NC, US"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"North Carolina State University,Department of Computer Science,Raleigh,NC,US","institution_ids":["https://openalex.org/I137902535"]},{"raw_affiliation_string":"Department of Computer Science, North Carolina State University, Raleigh, NC, US","institution_ids":["https://openalex.org/I137902535"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I137902535"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":"55","issue":null,"first_page":"1","last_page":"9"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.9908000230789185,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11652","display_name":"Imbalanced Data Classification Techniques","score":0.9872000217437744,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8592873811721802},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.7216535806655884},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.6602164506912231},{"id":"https://openalex.org/keywords/control","display_name":"Control (management)","score":0.5329243540763855},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5106950402259827},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5092240571975708},{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.4183042347431183},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.14273223280906677}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8592873811721802},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.7216535806655884},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.6602164506912231},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.5329243540763855},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5106950402259827},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5092240571975708},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.4183042347431183},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.14273223280906677},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/fie58773.2023.10343508","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/fie58773.2023.10343508","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE Frontiers in Education Conference (FIE)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":28,"referenced_works":["https://openalex.org/W136732505","https://openalex.org/W186099978","https://openalex.org/W240701929","https://openalex.org/W1555177075","https://openalex.org/W1966184253","https://openalex.org/W1969097043","https://openalex.org/W1970381522","https://openalex.org/W1975879668","https://openalex.org/W1983897914","https://openalex.org/W2019029324","https://openalex.org/W2036625304","https://openalex.org/W2045873471","https://openalex.org/W2053075547","https://openalex.org/W2053154970","https://openalex.org/W2078174348","https://openalex.org/W2133943399","https://openalex.org/W2155243985","https://openalex.org/W2201853392","https://openalex.org/W2296739333","https://openalex.org/W2595576831","https://openalex.org/W2918245745","https://openalex.org/W3021792676","https://openalex.org/W3121257585","https://openalex.org/W3205042302","https://openalex.org/W4200251786","https://openalex.org/W4239406995","https://openalex.org/W4292341621","https://openalex.org/W6680834967"],"related_works":["https://openalex.org/W2384667405","https://openalex.org/W2389266787","https://openalex.org/W2363885542","https://openalex.org/W2387206255","https://openalex.org/W2355570285","https://openalex.org/W2364958354","https://openalex.org/W2352797113","https://openalex.org/W2372137583","https://openalex.org/W2374872591","https://openalex.org/W1548050717"],"abstract_inverted_index":{"The":[0,138],"effectiveness":[1,59,188],"of":[2,9,29,60,71,124,175,193],"machine":[3,82],"learning":[4],"depends":[5],"on":[6,56,178],"the":[7,10,16,25,58,66,69,106,122,173,191],"quality":[8,48,70,118,141],"training":[11,17,103],"data.":[12],"In":[13],"some":[14],"fields,":[15],"data":[18,179],"is":[19,32],"generated":[20],"from":[21,105],"crowdsourced":[22],"labels.":[23,196],"Ensuring":[24],"reliability":[26,89,192],"and":[27,45,62,68,81,94,100,127,162,171],"accuracy":[28],"crowd-generated":[30],"labels":[31],"a":[33],"critical":[34],"challenge.":[35],"To":[36],"address":[37],"this,":[38],"we":[39,185],"present":[40],"comprehensive":[41],"research":[42,111],"to":[43,87,91,98],"develop":[44],"implement":[46],"robust":[47],"control":[49,119,142],"strategies":[50,120,143],"in":[51,121,189],"crowd":[52,194],"labeling.":[53],"We":[54],"focus":[55],"enhancing":[57,190],"feedback":[61],"suggestions":[63],"by":[64,75,115,136],"evaluating":[65],"taggers":[67,146,152,158],"tags":[72],"they":[73],"assign":[74,88],"using":[76],"natural":[77],"language":[78],"processing":[79],"techniques":[80],"learning.":[83],"Our":[84],"approach":[85],"aims":[86],"metrics":[90],"each":[92,176],"tagger":[93],"tag,":[95],"enabling":[96],"researchers":[97],"filter":[99],"create":[101],"machine-learning":[102],"datasets":[104],"most":[107],"reliable":[108],"annotations.":[109],"This":[110],"addresses":[112],"this":[113],"issue":[114],"implementing":[116,170],"four":[117,139],"domain":[123],"peer":[125],"assessment":[126],"comparing":[128],"their":[129,187],"performance":[130],"with":[131,181],"manual":[132,182],"grade":[133,183],"scores":[134],"assigned":[135],"professors.":[137],"key":[140],"encompass":[144],"identifying":[145],"who":[147],"tag":[148],"too":[149],"quickly,":[150],"detecting":[151],"providing":[153],"inconsistent":[154],"labels,":[155],"uncovering":[156],"unreliable":[157],"employing":[159],"pattern-based":[160],"tagging,":[161],"performing":[163],"agreement/disagreement":[164],"analysis":[165],"for":[166],"tags.":[167],"By":[168],"individually":[169],"assessing":[172],"impact":[174],"strategy":[177],"alignment":[180],"scores,":[184],"ascertain":[186],"-generated":[195]},"counts_by_year":[{"year":2024,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}