{"id":"https://openalex.org/W4406459903","doi":"https://doi.org/10.1109/ithet61869.2024.10837645","title":"Appropriate Number of Raters for IRT Based Peer Assessment Evaluation of Programming Skills","display_name":"Appropriate Number of Raters for IRT Based Peer Assessment Evaluation of Programming Skills","publication_year":2024,"publication_date":"2024-11-06","ids":{"openalex":"https://openalex.org/W4406459903","doi":"https://doi.org/10.1109/ithet61869.2024.10837645"},"language":"en","primary_location":{"id":"doi:10.1109/ithet61869.2024.10837645","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ithet61869.2024.10837645","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 21st International Conference on Information Technology Based Higher Education and Training (ITHET)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5070484773","display_name":"Minoru Nakayama","orcid":"https://orcid.org/0000-0001-5563-6901"},"institutions":[{"id":"https://openalex.org/I114531698","display_name":"Tokyo Institute of Technology","ror":"https://ror.org/0112mx960","country_code":"JP","type":"education","lineage":["https://openalex.org/I114531698"]},{"id":"https://openalex.org/I4400009020","display_name":"Institute of Science Tokyo","ror":"https://ror.org/05dqf9946","country_code":null,"type":"education","lineage":["https://openalex.org/I4400009020"]}],"countries":["JP"],"is_corresponding":true,"raw_author_name":"Minoru Nakayama","raw_affiliation_strings":["Institute of Science Tokyo (Tokyo Tech.) Meguro-ku,Tokyo,Japan"],"affiliations":[{"raw_affiliation_string":"Institute of Science Tokyo (Tokyo Tech.) Meguro-ku,Tokyo,Japan","institution_ids":["https://openalex.org/I114531698","https://openalex.org/I4400009020"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060310626","display_name":"Masaki Uto","orcid":"https://orcid.org/0000-0002-9330-5158"},"institutions":[{"id":"https://openalex.org/I20529979","display_name":"University of Electro-Communications","ror":"https://ror.org/02x73b849","country_code":"JP","type":"education","lineage":["https://openalex.org/I20529979"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Masaki Uto","raw_affiliation_strings":["University of Electro-Communications Chofu,Tokyo,Japan"],"affiliations":[{"raw_affiliation_string":"University of Electro-Communications Chofu,Tokyo,Japan","institution_ids":["https://openalex.org/I20529979"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5013432946","display_name":"Marco Temperini","orcid":null},"institutions":[{"id":"https://openalex.org/I861853513","display_name":"Sapienza University of Rome","ror":"https://ror.org/02be6w209","country_code":"IT","type":"education","lineage":["https://openalex.org/I861853513"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Marco Temperini","raw_affiliation_strings":["Sapienza University,Rome,Italy"],"affiliations":[{"raw_affiliation_string":"Sapienza University,Rome,Italy","institution_ids":["https://openalex.org/I861853513"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5032210048","display_name":"Filippo Sciarrone","orcid":"https://orcid.org/0000-0002-9631-8528"},"institutions":[{"id":"https://openalex.org/I4210132817","display_name":"Mercatorum University","ror":"https://ror.org/03znjxt55","country_code":"IT","type":"education","lineage":["https://openalex.org/I4210132817"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Filippo Sciarrone","raw_affiliation_strings":["Universitas Mercatorum,Rome,Italy"],"affiliations":[{"raw_affiliation_string":"Universitas Mercatorum,Rome,Italy","institution_ids":["https://openalex.org/I4210132817"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5070484773"],"corresponding_institution_ids":["https://openalex.org/I114531698","https://openalex.org/I4400009020"],"apc_list":null,"apc_paid":null,"fwci":2.1735,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.90888511,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"5"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10959","display_name":"Student Assessment and Feedback","score":0.9794999957084656,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10959","display_name":"Student Assessment and Feedback","score":0.9794999957084656,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T14025","display_name":"Educational Technology and Assessment","score":0.9746000170707703,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11902","display_name":"Intelligent Tutoring Systems and Adaptive Learning","score":0.951200008392334,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7302610874176025},{"id":"https://openalex.org/keywords/peer-assessment","display_name":"Peer assessment","score":0.6616430282592773},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.38003313541412354},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3208329677581787},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.24236217141151428},{"id":"https://openalex.org/keywords/mathematics-education","display_name":"Mathematics education","score":0.15212705731391907}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7302610874176025},{"id":"https://openalex.org/C53839665","wikidata":"https://www.wikidata.org/wiki/Q2067088","display_name":"Peer assessment","level":2,"score":0.6616430282592773},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.38003313541412354},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3208329677581787},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.24236217141151428},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.15212705731391907}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/ithet61869.2024.10837645","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ithet61869.2024.10837645","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 21st International Conference on Information Technology Based Higher Education and Training (ITHET)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.8100000023841858,"display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":16,"referenced_works":["https://openalex.org/W120343928","https://openalex.org/W1501652523","https://openalex.org/W2558594396","https://openalex.org/W2784788736","https://openalex.org/W2801332848","https://openalex.org/W2808968142","https://openalex.org/W2888255487","https://openalex.org/W2912139092","https://openalex.org/W3023028398","https://openalex.org/W3049260078","https://openalex.org/W3136931152","https://openalex.org/W4226353990","https://openalex.org/W4255493720","https://openalex.org/W4313067746","https://openalex.org/W4318969502","https://openalex.org/W6763769013"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W3204019825"],"abstract_inverted_index":{"In":[0],"order":[1],"to":[2,113],"examine":[3],"the":[4,11,20,30,35,40,50,71,84,99,110,118,140],"reliability":[5],"of":[6,13,37,42,57,64,70,86,120,132,139,142],"peer":[7,17,67,87,121,137],"assessment":[8,18,138],"settings":[9],"for":[10],"evaluation":[12,141],"programming":[14],"skills":[15,144],"using":[16,46,93,109],"and":[19,39,104,123],"item":[21],"response":[22],"theory":[23],"(IRT)":[24],"technique":[25],"in":[26,117,145],"a":[27,130],"small":[28],"classes,":[29],"optimal":[31],"conditions":[32,81],"such":[33,82],"as":[34,83],"numbers":[36],"peers":[38],"number":[41,85,119],"tasks":[43,90],"are":[44,91],"investigated":[45],"parameters":[47],"extracted":[48],"from":[49],"surveyed":[51],"data.":[52],"The":[53,125],"survey":[54],"data":[55],"consisted":[56,63],"31":[58],"students":[59],"whose":[60],"partial":[61],"participation":[62],"joining":[65],"three":[66],"assessments":[68],"out":[69],"5":[72],"sessions":[73],"during":[74],"which":[75],"these":[76],"took":[77],"place.":[78],"Peer":[79],"rating":[80],"raters":[88,122],"or":[89],"examined":[92,108],"mean":[94],"expected":[95],"standard":[96],"errors.":[97],"Also,":[98],"relationship":[100],"between":[101],"instructor's":[102],"ratings":[103],"estimated":[105],"ability":[106],"is":[107],"IRT":[111],"model":[112],"look":[114],"at":[115],"variations":[116],"tasks.":[124],"results":[126],"provide":[127],"evidence":[128],"that":[129],"set":[131],"guidelines":[133],"could":[134],"better":[135],"organise":[136],"programmina":[143],"actual":[146],"course":[147],"settings,":[148]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}