{"id":"https://openalex.org/W95021943","doi":null,"title":"Assessing the Reliability of Peer Evaluation of Undergraduate Research Papers Through the Use of Generalizability Theory","display_name":"Assessing the Reliability of Peer Evaluation of Undergraduate Research Papers Through the Use of Generalizability Theory","publication_year":2012,"publication_date":"2012-08-30","ids":{"openalex":"https://openalex.org/W95021943","mag":"95021943"},"language":"en","primary_location":{"id":"mag:95021943","doi":null,"is_oa":true,"landing_page_url":"https://papers.ssrn.com/sol3/Delivery.cfm/SSRN_ID2141166_code1511711.pdf?abstractid=2104478&mirid=5","pdf_url":null,"source":{"id":"https://openalex.org/S4210172589","display_name":"SSRN Electronic Journal","issn_l":"1556-5068","issn":["1556-5068"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1318003438","host_organization_name":"RELX Group (Netherlands)","host_organization_lineage":["https://openalex.org/I1318003438"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"SSRN Electronic Journal","raw_type":null},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://papers.ssrn.com/sol3/Delivery.cfm/SSRN_ID2141166_code1511711.pdf?abstractid=2104478&mirid=5","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5078826177","display_name":"Mihaiela Ristei Gugiu","orcid":"https://orcid.org/0000-0003-1554-3123","observed_orcids":["https://orcid.org/0000-0003-1554-3123"]},"institutions":[{"id":"https://openalex.org/I52357470","display_name":"The Ohio State University","ror":"https://ror.org/00rs6vg23","country_code":"US","type":"education","lineage":["https://openalex.org/I52357470"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mihaiela Ristei Gugiu","raw_affiliation_strings":["the Ohio State University"],"raw_orcid":"https://orcid.org/0000-0003-1554-3123","affiliations":[{"raw_affiliation_string":"the Ohio State University","institution_ids":["https://openalex.org/I52357470"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5073979944","display_name":"P. Cristian Gugiu","orcid":"https://orcid.org/0000-0003-0022-287X","observed_orcids":["https://orcid.org/0000-0003-0022-287X"]},"institutions":[{"id":"https://openalex.org/I52357470","display_name":"The Ohio State University","ror":"https://ror.org/00rs6vg23","country_code":"US","type":"education","lineage":["https://openalex.org/I52357470"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"P. Cristian Gugiu","raw_affiliation_strings":["Ohio State University (OSU) - College of Education and Human Ecology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Ohio State University (OSU) - College of Education and Human Ecology","institution_ids":["https://openalex.org/I52357470"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I52357470"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10959","display_name":"Student Assessment and Feedback","score":0.8848000168800354,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10959","display_name":"Student Assessment and Feedback","score":0.8848000168800354,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.06459999829530716,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T14025","display_name":"Educational Technology and Assessment","score":0.005100000184029341,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/peer-assessment","display_name":"peer assessment","score":0.9739999771118164},{"id":"https://openalex.org/keywords/generalizability-theory","display_name":"generalizability theory","score":0.8140000104904175},{"id":"https://openalex.org/keywords/reliability-statistics","display_name":"reliability statistics","score":0.6660000085830688},{"id":"https://openalex.org/keywords/validity","display_name":"validity","score":0.9710000157356262},{"id":"https://openalex.org/keywords/essay-scoring","display_name":"essay scoring","score":0.9440000057220459},{"id":"https://openalex.org/keywords/writing-instruction","display_name":"writing instruction","score":0.4309999942779541},{"id":"https://openalex.org/keywords/faculty-workload","display_name":"faculty workload","score":0.6460000276565552}],"concepts":[{"id":"https://openalex.org/C27158222","wikidata":"https://www.wikidata.org/wiki/Q5532422","display_name":"Generalizability theory","level":2,"score":0.9419722557067871},{"id":"https://openalex.org/C2777286243","wikidata":"https://www.wikidata.org/wiki/Q5591926","display_name":"Grading (engineering)","level":2,"score":0.6726877093315125},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.6342705488204956},{"id":"https://openalex.org/C2778476105","wikidata":"https://www.wikidata.org/wiki/Q628539","display_name":"Workload","level":2,"score":0.5707302093505859},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.556844174861908},{"id":"https://openalex.org/C70364389","wikidata":"https://www.wikidata.org/wiki/Q18757","display_name":"Validity","level":3,"score":0.4665868878364563},{"id":"https://openalex.org/C53839665","wikidata":"https://www.wikidata.org/wiki/Q2067088","display_name":"Peer assessment","level":2,"score":0.4396597146987915},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.42587384581565857},{"id":"https://openalex.org/C2779356329","wikidata":"https://www.wikidata.org/wiki/Q922625","display_name":"Checklist","level":2,"score":0.42054662108421326},{"id":"https://openalex.org/C75630572","wikidata":"https://www.wikidata.org/wiki/Q538904","display_name":"Applied psychology","level":1,"score":0.33073943853378296},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.27769866585731506},{"id":"https://openalex.org/C171606756","wikidata":"https://www.wikidata.org/wiki/Q506132","display_name":"Psychometrics","level":2,"score":0.15975266695022583},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.14352747797966003},{"id":"https://openalex.org/C70410870","wikidata":"https://www.wikidata.org/wiki/Q199906","display_name":"Clinical psychology","level":1,"score":0.14062464237213135},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.1280132532119751},{"id":"https://openalex.org/C138496976","wikidata":"https://www.wikidata.org/wiki/Q175002","display_name":"Developmental psychology","level":1,"score":0.08202815055847168},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C147176958","wikidata":"https://www.wikidata.org/wiki/Q77590","display_name":"Civil engineering","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"mag:95021943","doi":null,"is_oa":true,"landing_page_url":"https://papers.ssrn.com/sol3/Delivery.cfm/SSRN_ID2141166_code1511711.pdf?abstractid=2104478&mirid=5","pdf_url":null,"source":{"id":"https://openalex.org/S4210172589","display_name":"SSRN Electronic Journal","issn_l":"1556-5068","issn":["1556-5068"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1318003438","host_organization_name":"RELX Group (Netherlands)","host_organization_lineage":["https://openalex.org/I1318003438"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"SSRN Electronic Journal","raw_type":null}],"best_oa_location":{"id":"mag:95021943","doi":null,"is_oa":true,"landing_page_url":"https://papers.ssrn.com/sol3/Delivery.cfm/SSRN_ID2141166_code1511711.pdf?abstractid=2104478&mirid=5","pdf_url":null,"source":{"id":"https://openalex.org/S4210172589","display_name":"SSRN Electronic Journal","issn_l":"1556-5068","issn":["1556-5068"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1318003438","host_organization_name":"RELX Group (Netherlands)","host_organization_lineage":["https://openalex.org/I1318003438"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"SSRN Electronic Journal","raw_type":null},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.8278999924659729,"display_name":"Quality education"}],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":30,"referenced_works":["https://openalex.org/W99897896","https://openalex.org/W243551322","https://openalex.org/W1496437928","https://openalex.org/W1543793426","https://openalex.org/W1834160718","https://openalex.org/W1928697341","https://openalex.org/W1964052265","https://openalex.org/W1979545165","https://openalex.org/W1991482315","https://openalex.org/W1995939719","https://openalex.org/W1996469193","https://openalex.org/W2005112578","https://openalex.org/W2006996167","https://openalex.org/W2008873917","https://openalex.org/W2024583961","https://openalex.org/W2031703645","https://openalex.org/W2037091100","https://openalex.org/W2048679426","https://openalex.org/W2050220980","https://openalex.org/W2055699422","https://openalex.org/W2071784741","https://openalex.org/W2077649418","https://openalex.org/W2080170979","https://openalex.org/W2084605636","https://openalex.org/W2140661853","https://openalex.org/W2141403362","https://openalex.org/W2148189517","https://openalex.org/W2151242492","https://openalex.org/W2159306398","https://openalex.org/W2612259620"],"related_works":["https://openalex.org/W2564182289","https://openalex.org/W2140661853","https://openalex.org/W2933516508","https://openalex.org/W1576901098","https://openalex.org/W2005932354","https://openalex.org/W3138239805","https://openalex.org/W2091462764","https://openalex.org/W4407670471","https://openalex.org/W2620041690","https://openalex.org/W15536309"],"abstract_inverted_index":{"Educational":[0],"researchers":[1],"have":[2,29],"long":[3],"espoused":[4],"the":[5,19,50,64,112],"virtues":[6],"of":[7,21,54,66],"writing":[8],"with":[9],"regard":[10],"to":[11,85,92],"student":[12],"cognitive":[13],"skills.":[14],"However,":[15],"unlike":[16],"objective":[17],"tests,":[18],"grading":[20],"essays":[22],"is":[23],"time":[24],"consuming.":[25],"Therefore,":[26],"peer":[27,55,67,98],"evaluations":[28,56],"become":[30],"an":[31],"increasingly":[32],"common":[33],"method":[34,109],"by":[35],"which":[36],"faculty":[37],"can":[38],"integrate":[39],"written":[40],"assignments":[41],"without":[42],"substantially":[43],"increasing":[44],"their":[45],"workload.":[46],"Although":[47],"research":[48],"into":[49],"reliability":[51,87,115],"and":[52,88],"validity":[53,104],"has":[57],"found":[58],"mixed":[59],"results,":[60],"most":[61],"evidence":[62],"supports":[63],"use":[65],"evaluations.":[68],"Unfortunately,":[69],"these":[70],"studies":[71],"frequently":[72],"committed":[73],"one":[74],"or":[75],"more":[76],"analytical":[77],"flaws.":[78],"The":[79,95],"present":[80],"study":[81],"employed":[82],"Generalizability":[83],"Theory":[84],"estimate":[86],"a":[89],"paired":[90],"t-test":[91],"assess":[93],"validity.":[94],"results":[96],"showed":[97],"evaluation":[99],"was":[100,105,116],"reliable":[101],"but":[102],"its":[103],"questionable.":[106],"A":[107],"new":[108],"for":[110],"computing":[111],"minimum":[113],"acceptable":[114],"also":[117],"introduced.":[118]},"counts_by_year":[],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2016-06-24T00:00:00"}