{"id":"https://openalex.org/W3005175757","doi":"https://doi.org/10.1109/tlt.2020.2971495","title":"The Effects of Dispersion and Reciprocity on Assessment Fidelity in Peer-Review Systems: A Simulation Study","display_name":"The Effects of Dispersion and Reciprocity on Assessment Fidelity in Peer-Review Systems: A Simulation Study","publication_year":2020,"publication_date":"2020-02-04","ids":{"openalex":"https://openalex.org/W3005175757","doi":"https://doi.org/10.1109/tlt.2020.2971495","mag":"3005175757"},"language":"en","primary_location":{"id":"doi:10.1109/tlt.2020.2971495","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tlt.2020.2971495","pdf_url":null,"source":{"id":"https://openalex.org/S130363450","display_name":"IEEE Transactions on Learning Technologies","issn_l":"1939-1382","issn":["1939-1382","2372-0050"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Learning Technologies","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5086173007","display_name":"Dmytro Babik","orcid":"https://orcid.org/0000-0001-6464-6028"},"institutions":[{"id":"https://openalex.org/I11883440","display_name":"James Madison University","ror":"https://ror.org/028pmsz77","country_code":"US","type":"education","lineage":["https://openalex.org/I11883440"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dmytro Babik","raw_affiliation_strings":["James Madison University, Harrisonburg, USA"],"raw_orcid":"https://orcid.org/0000-0001-6464-6028","affiliations":[{"raw_affiliation_string":"James Madison University, Harrisonburg, USA","institution_ids":["https://openalex.org/I11883440"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032565635","display_name":"Scott P. Stevens","orcid":"https://orcid.org/0000-0003-4691-4599"},"institutions":[{"id":"https://openalex.org/I11883440","display_name":"James Madison University","ror":"https://ror.org/028pmsz77","country_code":"US","type":"education","lineage":["https://openalex.org/I11883440"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Scott P. Stevens","raw_affiliation_strings":["James Madison University, Harrisonburg, USA"],"raw_orcid":"https://orcid.org/0000-0003-4691-4599","affiliations":[{"raw_affiliation_string":"James Madison University, Harrisonburg, USA","institution_ids":["https://openalex.org/I11883440"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061631706","display_name":"Andrew E. Waters","orcid":null},"institutions":[{"id":"https://openalex.org/I74775410","display_name":"Rice University","ror":"https://ror.org/008zs3103","country_code":"US","type":"education","lineage":["https://openalex.org/I74775410"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Andrew Waters","raw_affiliation_strings":["OpenStax / Rice University, Houston, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"OpenStax / Rice University, Houston, USA","institution_ids":["https://openalex.org/I74775410"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5060792994","display_name":"David Tinapple","orcid":"https://orcid.org/0000-0002-8493-0023"},"institutions":[{"id":"https://openalex.org/I55732556","display_name":"Arizona State University","ror":"https://ror.org/03efmqc40","country_code":"US","type":"education","lineage":["https://openalex.org/I55732556"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"David Tinapple","raw_affiliation_strings":["Arizona State University, Tempe, USA"],"raw_orcid":"https://orcid.org/0000-0002-8493-0023","affiliations":[{"raw_affiliation_string":"Arizona State University, Tempe, USA","institution_ids":["https://openalex.org/I55732556"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.255,"has_fulltext":false,"cited_by_count":5,"citation_normalized_percentile":{"value":0.63236507,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":"13","issue":"3","first_page":"580","last_page":"592"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.9883999824523926,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.9883999824523926,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.9484000205993652,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12478","display_name":"Wikis in Education and Collaboration","score":0.9442999958992004,"subfield":{"id":"https://openalex.org/subfields/3315","display_name":"Communication"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8042072057723999},{"id":"https://openalex.org/keywords/fidelity","display_name":"Fidelity","score":0.6863735318183899},{"id":"https://openalex.org/keywords/reciprocity","display_name":"Reciprocity (cultural anthropology)","score":0.6411531567573547},{"id":"https://openalex.org/keywords/peer-to-peer","display_name":"Peer-to-peer","score":0.48874545097351074},{"id":"https://openalex.org/keywords/peer-assessment","display_name":"Peer assessment","score":0.45552903413772583},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.397991418838501},{"id":"https://openalex.org/keywords/multimedia","display_name":"Multimedia","score":0.36933284997940063},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.36158666014671326},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.1978129744529724},{"id":"https://openalex.org/keywords/telecommunications","display_name":"Telecommunications","score":0.12498387694358826},{"id":"https://openalex.org/keywords/mathematics-education","display_name":"Mathematics education","score":0.10957103967666626},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.1015813946723938}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8042072057723999},{"id":"https://openalex.org/C2776459999","wikidata":"https://www.wikidata.org/wiki/Q2119376","display_name":"Fidelity","level":2,"score":0.6863735318183899},{"id":"https://openalex.org/C169903001","wikidata":"https://www.wikidata.org/wiki/Q3264987","display_name":"Reciprocity (cultural anthropology)","level":2,"score":0.6411531567573547},{"id":"https://openalex.org/C534932454","wikidata":"https://www.wikidata.org/wiki/Q161410","display_name":"Peer-to-peer","level":2,"score":0.48874545097351074},{"id":"https://openalex.org/C53839665","wikidata":"https://www.wikidata.org/wiki/Q2067088","display_name":"Peer assessment","level":2,"score":0.45552903413772583},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.397991418838501},{"id":"https://openalex.org/C49774154","wikidata":"https://www.wikidata.org/wiki/Q131765","display_name":"Multimedia","level":1,"score":0.36933284997940063},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.36158666014671326},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.1978129744529724},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.12498387694358826},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.10957103967666626},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.1015813946723938},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tlt.2020.2971495","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tlt.2020.2971495","pdf_url":null,"source":{"id":"https://openalex.org/S130363450","display_name":"IEEE Transactions on Learning Technologies","issn_l":"1939-1382","issn":["1939-1382","2372-0050"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Learning Technologies","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.5600000023841858,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[{"id":"https://openalex.org/F4320310375","display_name":"James Madison University","ror":"https://ror.org/028pmsz77"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":100,"referenced_works":["https://openalex.org/W138614799","https://openalex.org/W142345412","https://openalex.org/W1260464102","https://openalex.org/W1504869719","https://openalex.org/W1511351087","https://openalex.org/W1518732113","https://openalex.org/W1574165943","https://openalex.org/W1707985454","https://openalex.org/W1919586704","https://openalex.org/W1964659143","https://openalex.org/W1970685234","https://openalex.org/W1970871468","https://openalex.org/W1982429765","https://openalex.org/W1983285030","https://openalex.org/W1984314602","https://openalex.org/W1989199392","https://openalex.org/W1992423495","https://openalex.org/W1995875735","https://openalex.org/W1999857980","https://openalex.org/W2004295642","https://openalex.org/W2015344683","https://openalex.org/W2019923146","https://openalex.org/W2024132711","https://openalex.org/W2028988099","https://openalex.org/W2036627610","https://openalex.org/W2036910311","https://openalex.org/W2043320673","https://openalex.org/W2045873471","https://openalex.org/W2052104835","https://openalex.org/W2058051197","https://openalex.org/W2064268350","https://openalex.org/W2065364488","https://openalex.org/W2065631266","https://openalex.org/W2069099927","https://openalex.org/W2070635790","https://openalex.org/W2070693116","https://openalex.org/W2077304088","https://openalex.org/W2078152917","https://openalex.org/W2079240032","https://openalex.org/W2086213192","https://openalex.org/W2088700588","https://openalex.org/W2091579301","https://openalex.org/W2093016541","https://openalex.org/W2095123227","https://openalex.org/W2098992709","https://openalex.org/W2103868989","https://openalex.org/W2114563525","https://openalex.org/W2116119142","https://openalex.org/W2119387217","https://openalex.org/W2122588206","https://openalex.org/W2126777454","https://openalex.org/W2133487567","https://openalex.org/W2134529236","https://openalex.org/W2136451344","https://openalex.org/W2140661853","https://openalex.org/W2143579043","https://openalex.org/W2146267631","https://openalex.org/W2148606196","https://openalex.org/W2148747005","https://openalex.org/W2151242492","https://openalex.org/W2157771935","https://openalex.org/W2157952045","https://openalex.org/W2158346874","https://openalex.org/W2160449301","https://openalex.org/W2160756447","https://openalex.org/W2166164009","https://openalex.org/W2167330974","https://openalex.org/W2244249936","https://openalex.org/W2286107653","https://openalex.org/W2325350314","https://openalex.org/W2332237006","https://openalex.org/W2332347705","https://openalex.org/W2339622404","https://openalex.org/W2339932976","https://openalex.org/W2411217907","https://openalex.org/W2469893666","https://openalex.org/W2579555219","https://openalex.org/W2607400135","https://openalex.org/W2759588426","https://openalex.org/W2802553787","https://openalex.org/W2949835649","https://openalex.org/W2963112694","https://openalex.org/W2963433558","https://openalex.org/W3021174912","https://openalex.org/W3122520279","https://openalex.org/W3151685851","https://openalex.org/W4230906437","https://openalex.org/W4232692970","https://openalex.org/W4239406995","https://openalex.org/W4241246521","https://openalex.org/W4298692417","https://openalex.org/W4388169926","https://openalex.org/W4390275381","https://openalex.org/W6675427288","https://openalex.org/W6677855075","https://openalex.org/W6703823026","https://openalex.org/W6714712443","https://openalex.org/W6736532276","https://openalex.org/W6776536710","https://openalex.org/W6860055233"],"related_works":["https://openalex.org/W2381850946","https://openalex.org/W4380449851","https://openalex.org/W3125091513","https://openalex.org/W3121820748","https://openalex.org/W164043708","https://openalex.org/W1734862528","https://openalex.org/W3124926124","https://openalex.org/W4248383205","https://openalex.org/W4234745530","https://openalex.org/W2146383839"],"abstract_inverted_index":{"Over":[0],"the":[1,24,56,69,72,106,109],"last":[2],"20":[3],"years,":[4],"online":[5,30],"peer":[6],"review":[7],"and":[8,14,29,34,52,85,128,147],"assessment":[9,132],"have":[10],"become":[11],"widely":[12],"used":[13],"well-researched":[15],"practices":[16],"in":[17,65,68,134,137],"education.":[18],"Their":[19],"use":[20],"increased,":[21],"especially":[22],"with":[23,139],"proliferation":[25],"of":[26,32,55,71,108,157],"nonconventional":[27],"large-scale":[28],"modes":[31],"teaching":[33],"learning,":[35],"such":[36,143],"as":[37,83,144],"Massive":[38],"Open":[39],"Online":[40],"Courses":[41],"(MOOCs).":[42],"A":[43],"well-designed":[44],"peer-review":[45,73,135,158],"system":[46],"is":[47],"expected":[48],"to":[49,80,103,114,122],"produce":[50],"valid":[51],"reliable":[53],"assessments":[54],"artifacts":[57],"created":[58],"by":[59],"participants.":[60],"These":[61],"systems":[62,136,159],"notably":[63],"vary":[64],"designs,":[66],"particularly":[67],"structure":[70],"networks,":[74],"i.e.,":[75],"how":[76,95],"participants":[77],"are":[78],"linked":[79],"each":[81],"other":[82,140],"creators":[84],"reviewers.":[86],"To":[87],"date,":[88],"little":[89],"research":[90],"has":[91],"been":[92],"done":[93],"on":[94],"different":[96],"network":[97,126],"structures":[98],"impact":[99],"a":[100,119],"system's":[101],"ability":[102],"accurately":[104],"assess":[105],"quality":[107],"evaluated":[110],"artifacts.":[111],"We":[112,150],"begin":[113],"address":[115],"this":[116],"gap,":[117],"using":[118],"simulation":[120],"approach":[121],"demonstrate":[123],"that":[124],"two":[125],"characteristics-dispersion":[127],"reciprocity-may":[129],"indeed":[130],"affect":[131],"fidelity":[133],"conjunction":[138],"design":[141],"choices,":[142],"evaluation":[145],"scale":[146],"aggregation":[148],"method.":[149],"also":[151],"outline":[152],"directions":[153],"for":[154],"further":[155],"investigations":[156],"design.":[160]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":1},{"year":2019,"cited_by_count":1}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}