{"id":"https://openalex.org/W4318969298","doi":"https://doi.org/10.1109/ithet56107.2022.10031993","title":"Estimating Student Grades through Peer Assessment as a Crowdsourcing Calibration Problem","display_name":"Estimating Student Grades through Peer Assessment as a Crowdsourcing Calibration Problem","publication_year":2022,"publication_date":"2022-11-07","ids":{"openalex":"https://openalex.org/W4318969298","doi":"https://doi.org/10.1109/ithet56107.2022.10031993"},"language":"en","primary_location":{"id":"doi:10.1109/ithet56107.2022.10031993","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/ithet56107.2022.10031993","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2022 20th International Conference on Information Technology Based Higher Education and Training (ITHET)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5084180743","display_name":"Yunkai Xiao","orcid":null},"institutions":[{"id":"https://openalex.org/I137902535","display_name":"North Carolina State University","ror":"https://ror.org/04tj63d06","country_code":"US","type":"education","lineage":["https://openalex.org/I137902535"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Yunkai Xiao","raw_affiliation_strings":["North Carolina State University,Department of Computer Science,Raleigh,NC,United States","Department of Computer Science, North Carolina State University, Raleigh, NC, United States"],"affiliations":[{"raw_affiliation_string":"North Carolina State University,Department of Computer Science,Raleigh,NC,United States","institution_ids":["https://openalex.org/I137902535"]},{"raw_affiliation_string":"Department of Computer Science, North Carolina State University, Raleigh, NC, United States","institution_ids":["https://openalex.org/I137902535"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102503525","display_name":"Yinan Gao","orcid":null},"institutions":[{"id":"https://openalex.org/I162868743","display_name":"Tianjin University","ror":"https://ror.org/012tb2g32","country_code":"CN","type":"education","lineage":["https://openalex.org/I162868743"]},{"id":"https://openalex.org/I193524592","display_name":"Tianjin University of Finance and Economics","ror":"https://ror.org/05ev1jb90","country_code":"CN","type":"education","lineage":["https://openalex.org/I193524592"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yinan Gao","raw_affiliation_strings":["Tianjin University,College of Management and Economics,Tianjin,China","College of Management and Economics, Tianjin University, Tianjin, China"],"affiliations":[{"raw_affiliation_string":"Tianjin University,College of Management and Economics,Tianjin,China","institution_ids":["https://openalex.org/I193524592"]},{"raw_affiliation_string":"College of Management and Economics, Tianjin University, Tianjin, China","institution_ids":["https://openalex.org/I162868743"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018268922","display_name":"Chuhuai Yue","orcid":null},"institutions":[{"id":"https://openalex.org/I125839683","display_name":"Beijing Institute of Technology","ror":"https://ror.org/01skt4w74","country_code":"CN","type":"education","lineage":["https://openalex.org/I125839683","https://openalex.org/I890469752"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chuhuai Yue","raw_affiliation_strings":["Beijing Institute of Technology,School of Computer Science and Technology,Beijing,China","School of Computer Science and Technology, Beijing Institute of Technology, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Beijing Institute of Technology,School of Computer Science and Technology,Beijing,China","institution_ids":["https://openalex.org/I125839683"]},{"raw_affiliation_string":"School of Computer Science and Technology, Beijing Institute of Technology, Beijing, China","institution_ids":["https://openalex.org/I125839683"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5040537321","display_name":"Edward F. Gehringer","orcid":"https://orcid.org/0000-0002-5217-2643"},"institutions":[{"id":"https://openalex.org/I137902535","display_name":"North Carolina State University","ror":"https://ror.org/04tj63d06","country_code":"US","type":"education","lineage":["https://openalex.org/I137902535"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Edward Gehringer","raw_affiliation_strings":["North Carolina State University,Department of Computer Science,Raleigh,NC,United States","Department of Computer Science, North Carolina State University, Raleigh, NC, United States"],"affiliations":[{"raw_affiliation_string":"North Carolina State University,Department of Computer Science,Raleigh,NC,United States","institution_ids":["https://openalex.org/I137902535"]},{"raw_affiliation_string":"Department of Computer Science, North Carolina State University, Raleigh, NC, United States","institution_ids":["https://openalex.org/I137902535"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5084180743"],"corresponding_institution_ids":["https://openalex.org/I137902535"],"apc_list":null,"apc_paid":null,"fwci":0.4586,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.73188629,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":95},"biblio":{"volume":"29","issue":null,"first_page":"1","last_page":"9"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9965999722480774,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9965999722480774,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9882000088691711,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12478","display_name":"Wikis in Education and Collaboration","score":0.9810000061988831,"subfield":{"id":"https://openalex.org/subfields/3315","display_name":"Communication"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.8325639963150024},{"id":"https://openalex.org/keywords/grading","display_name":"Grading (engineering)","score":0.764351487159729},{"id":"https://openalex.org/keywords/peer-assessment","display_name":"Peer assessment","score":0.7447411417961121},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7175924181938171},{"id":"https://openalex.org/keywords/strengths-and-weaknesses","display_name":"Strengths and weaknesses","score":0.6187399625778198},{"id":"https://openalex.org/keywords/trustworthiness","display_name":"Trustworthiness","score":0.5927872061729431},{"id":"https://openalex.org/keywords/peer-feedback","display_name":"Peer feedback","score":0.5637497305870056},{"id":"https://openalex.org/keywords/aside","display_name":"Aside","score":0.5077353715896606},{"id":"https://openalex.org/keywords/class","display_name":"Class (philosophy)","score":0.4518711566925049},{"id":"https://openalex.org/keywords/online-assessment","display_name":"Online assessment","score":0.42120304703712463},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3857693672180176},{"id":"https://openalex.org/keywords/mathematics-education","display_name":"Mathematics education","score":0.34861287474632263},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.24439787864685059},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.2273765206336975},{"id":"https://openalex.org/keywords/formative-assessment","display_name":"Formative assessment","score":0.1927039623260498},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.18046140670776367},{"id":"https://openalex.org/keywords/internet-privacy","display_name":"Internet privacy","score":0.16946107149124146},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.09476020932197571}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.8325639963150024},{"id":"https://openalex.org/C2777286243","wikidata":"https://www.wikidata.org/wiki/Q5591926","display_name":"Grading (engineering)","level":2,"score":0.764351487159729},{"id":"https://openalex.org/C53839665","wikidata":"https://www.wikidata.org/wiki/Q2067088","display_name":"Peer assessment","level":2,"score":0.7447411417961121},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7175924181938171},{"id":"https://openalex.org/C63882131","wikidata":"https://www.wikidata.org/wiki/Q17122954","display_name":"Strengths and weaknesses","level":2,"score":0.6187399625778198},{"id":"https://openalex.org/C153701036","wikidata":"https://www.wikidata.org/wiki/Q659974","display_name":"Trustworthiness","level":2,"score":0.5927872061729431},{"id":"https://openalex.org/C42058189","wikidata":"https://www.wikidata.org/wiki/Q7160400","display_name":"Peer feedback","level":2,"score":0.5637497305870056},{"id":"https://openalex.org/C2778120072","wikidata":"https://www.wikidata.org/wiki/Q2858150","display_name":"Aside","level":2,"score":0.5077353715896606},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.4518711566925049},{"id":"https://openalex.org/C178823583","wikidata":"https://www.wikidata.org/wiki/Q3269883","display_name":"Online assessment","level":3,"score":0.42120304703712463},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3857693672180176},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.34861287474632263},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.24439787864685059},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.2273765206336975},{"id":"https://openalex.org/C42525527","wikidata":"https://www.wikidata.org/wiki/Q1209955","display_name":"Formative assessment","level":2,"score":0.1927039623260498},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.18046140670776367},{"id":"https://openalex.org/C108827166","wikidata":"https://www.wikidata.org/wiki/Q175975","display_name":"Internet privacy","level":1,"score":0.16946107149124146},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.09476020932197571},{"id":"https://openalex.org/C124952713","wikidata":"https://www.wikidata.org/wiki/Q8242","display_name":"Literature","level":1,"score":0.0},{"id":"https://openalex.org/C142362112","wikidata":"https://www.wikidata.org/wiki/Q735","display_name":"Art","level":0,"score":0.0},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.0},{"id":"https://openalex.org/C147176958","wikidata":"https://www.wikidata.org/wiki/Q77590","display_name":"Civil engineering","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/ithet56107.2022.10031993","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/ithet56107.2022.10031993","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2022 20th International Conference on Information Technology Based Higher Education and Training (ITHET)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/9","display_name":"Industry, innovation and infrastructure","score":0.5600000023841858}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":51,"referenced_works":["https://openalex.org/W2637294","https://openalex.org/W9014458","https://openalex.org/W47203835","https://openalex.org/W60106149","https://openalex.org/W1593930920","https://openalex.org/W1595685357","https://openalex.org/W1999735680","https://openalex.org/W2002537601","https://openalex.org/W2050996878","https://openalex.org/W2085091872","https://openalex.org/W2090513801","https://openalex.org/W2102271161","https://openalex.org/W2121856130","https://openalex.org/W2144290841","https://openalex.org/W2168098237","https://openalex.org/W2170435532","https://openalex.org/W2176279353","https://openalex.org/W2189548424","https://openalex.org/W2251818274","https://openalex.org/W2337430557","https://openalex.org/W2795596428","https://openalex.org/W2899689163","https://openalex.org/W2901115780","https://openalex.org/W2921976894","https://openalex.org/W2948690651","https://openalex.org/W2965201228","https://openalex.org/W2996523803","https://openalex.org/W3001438767","https://openalex.org/W3010646643","https://openalex.org/W3024484017","https://openalex.org/W3028785035","https://openalex.org/W3052814785","https://openalex.org/W3112713339","https://openalex.org/W3120468046","https://openalex.org/W3124686198","https://openalex.org/W3135454619","https://openalex.org/W3162001182","https://openalex.org/W3196116149","https://openalex.org/W3214637961","https://openalex.org/W4200518127","https://openalex.org/W4248871535","https://openalex.org/W4284889946","https://openalex.org/W4386566648","https://openalex.org/W4398775968","https://openalex.org/W6600112240","https://openalex.org/W6601886427","https://openalex.org/W6635717402","https://openalex.org/W6679959949","https://openalex.org/W6691296735","https://openalex.org/W6772100259","https://openalex.org/W6777326601"],"related_works":["https://openalex.org/W2887619904","https://openalex.org/W2049130877","https://openalex.org/W4308390377","https://openalex.org/W2944962532","https://openalex.org/W2187381844","https://openalex.org/W2358388664","https://openalex.org/W3189646254","https://openalex.org/W4244408479","https://openalex.org/W2088700588","https://openalex.org/W4318969298"],"abstract_inverted_index":{"There":[0],"is":[1,15,22,71],"a":[2,28],"trend":[3],"to":[4,84,138,186],"move":[5],"education":[6,37,77],"into":[7],"an":[8,152,164,177],"online":[9,36,76],"environment,":[10],"especially":[11],"when":[12,68],"offline":[13],"learning":[14],"restricted":[16],"by":[17,24,124],"time,":[18],"space,":[19],"availability,":[20],"or":[21,115],"impacted":[23],"issues":[25],"such":[26],"as":[27],"public":[29],"health":[30],"incident.":[31],"Evaluating":[32],"students\u2019":[33,55,167],"performance":[34],"in":[35,146],"has":[38,104],"always":[39],"been":[40,80,136],"challenging.":[41],"Objective":[42],"questions,":[43],"which":[44],"can":[45],"be":[46,128],"graded":[47],"automatically,":[48],"could":[49,159],"only":[50],"assess":[51],"certain":[52,180],"aspects":[53],"of":[54,57,166,179],"mastery":[56],"knowledge.":[58],"A":[59],"grading":[60],"problem":[61],"appears":[62],"if":[63],"subjective":[64],"questions":[65],"exist,":[66],"primarily":[67],"the":[69,112,121,147,155],"class":[70],"taught":[72],"at":[73,130],"scale.":[74],"Many":[75,133],"platforms":[78],"have":[79,111,135],"using":[81],"peer":[82,91,102,170],"assessment":[83,92,103],"resolve":[85],"this":[86],"problem.":[87],"Aside":[88],"from":[89,169],"that,":[90],"also":[93],"improves":[94],"interactions":[95],"between":[96],"students,":[97],"instructors,":[98],"and":[99,120,142,172,190],"peers.":[100],"While":[101],"some":[105],"inherent":[106],"weaknesses,":[107],"reviewers":[108,174],"may":[109],"not":[110,127],"same":[113,156],"ability":[114],"attitude":[116],"toward":[117],"reviewing":[118],"others,":[119],"feedback":[122],"generated":[123],"them":[125],"shall":[126],"taken":[129],"face":[131],"value.":[132],"algorithms":[134],"developed":[137],"evaluate":[139],"annotators\u2019":[140],"trustworthiness":[141],"generate":[143],"reliable":[144],"labels":[145],"crowdsourcing":[148],"industry.":[149],"We":[150],"proposed":[151],"algorithm":[153],"under":[154],"concept":[157],"that":[158],"provide":[160],"accurate":[161],"automated":[162],"grading,":[163],"overview":[165],"weaknesses":[168],"feedback,":[171],"identify":[173],"who":[175],"lack":[176],"understanding":[178],"concepts.":[181],"This":[182],"information":[183],"allows":[184],"instructors":[185],"offer":[187],"targeted":[188],"training":[189],"create":[191],"data-driven":[192],"lesson":[193],"plans.":[194]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1}],"updated_date":"2025-12-24T23:09:58.560324","created_date":"2025-10-10T00:00:00"}