{"id":"https://openalex.org/W4293080803","doi":"https://doi.org/10.1162/qss_a_00207","title":"Peer reviewer topic choice and its impact on interrater reliability: A mixed-method study","display_name":"Peer reviewer topic choice and its impact on interrater reliability: A mixed-method study","publication_year":2022,"publication_date":"2022-01-01","ids":{"openalex":"https://openalex.org/W4293080803","doi":"https://doi.org/10.1162/qss_a_00207"},"language":"en","primary_location":{"id":"doi:10.1162/qss_a_00207","is_oa":true,"landing_page_url":"https://doi.org/10.1162/qss_a_00207","pdf_url":"https://direct.mit.edu/qss/article-pdf/doi/10.1162/qss_a_00207/2052976/qss_a_00207.pdf","source":{"id":"https://openalex.org/S4210195326","display_name":"Quantitative Science Studies","issn_l":"2641-3337","issn":["2641-3337"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310315718","host_organization_name":"The MIT Press","host_organization_lineage":["https://openalex.org/P4310315718"],"host_organization_lineage_names":["The MIT Press"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Quantitative Science Studies","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://direct.mit.edu/qss/article-pdf/doi/10.1162/qss_a_00207/2052976/qss_a_00207.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5044445824","display_name":"Thomas Feliciani","orcid":"https://orcid.org/0000-0003-4977-0877"},"institutions":[{"id":"https://openalex.org/I100930933","display_name":"University College Dublin","ror":"https://ror.org/05m7pjf47","country_code":"IE","type":"education","lineage":["https://openalex.org/I100930933"]}],"countries":["IE"],"is_corresponding":true,"raw_author_name":"Thomas Feliciani","raw_affiliation_strings":["School of Sociology and Geary Institute for Public Policy, University College Dublin, Dublin, Ireland"],"affiliations":[{"raw_affiliation_string":"School of Sociology and Geary Institute for Public Policy, University College Dublin, Dublin, Ireland","institution_ids":["https://openalex.org/I100930933"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5013476840","display_name":"Junwen Luo","orcid":"https://orcid.org/0000-0003-3347-3982"},"institutions":[{"id":"https://openalex.org/I100930933","display_name":"University College Dublin","ror":"https://ror.org/05m7pjf47","country_code":"IE","type":"education","lineage":["https://openalex.org/I100930933"]}],"countries":["IE"],"is_corresponding":false,"raw_author_name":"Junwen Luo","raw_affiliation_strings":["School of Information and Communication Studies and Geary Institute for Public Policy, University College Dublin, Dublin, Ireland"],"affiliations":[{"raw_affiliation_string":"School of Information and Communication Studies and Geary Institute for Public Policy, University College Dublin, Dublin, Ireland","institution_ids":["https://openalex.org/I100930933"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5036452490","display_name":"Kalpana Shankar","orcid":"https://orcid.org/0000-0001-8788-466X"},"institutions":[{"id":"https://openalex.org/I100930933","display_name":"University College Dublin","ror":"https://ror.org/05m7pjf47","country_code":"IE","type":"education","lineage":["https://openalex.org/I100930933"]}],"countries":["IE"],"is_corresponding":false,"raw_author_name":"Kalpana Shankar","raw_affiliation_strings":["School of Information and Communication Studies and Geary Institute for Public Policy, University College Dublin, Dublin, Ireland"],"affiliations":[{"raw_affiliation_string":"School of Information and Communication Studies and Geary Institute for Public Policy, University College Dublin, Dublin, Ireland","institution_ids":["https://openalex.org/I100930933"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5044445824"],"corresponding_institution_ids":["https://openalex.org/I100930933"],"apc_list":{"value":800,"currency":"USD","value_usd":800},"apc_paid":{"value":800,"currency":"USD","value_usd":800},"fwci":1.5901,"has_fulltext":false,"cited_by_count":6,"citation_normalized_percentile":{"value":0.86967789,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":"3","issue":"3","first_page":"832","last_page":"856"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.9848999977111816,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.9848999977111816,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/inter-rater-reliability","display_name":"Inter-rater reliability","score":0.9097204208374023},{"id":"https://openalex.org/keywords/diversity","display_name":"Diversity (politics)","score":0.5886318683624268},{"id":"https://openalex.org/keywords/agency","display_name":"Agency (philosophy)","score":0.5701032280921936},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.5355631709098816},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.5023181438446045},{"id":"https://openalex.org/keywords/peer-review","display_name":"Peer review","score":0.47959744930267334},{"id":"https://openalex.org/keywords/actuarial-science","display_name":"Actuarial science","score":0.4090949296951294},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.24185344576835632},{"id":"https://openalex.org/keywords/sociology","display_name":"Sociology","score":0.1959158182144165},{"id":"https://openalex.org/keywords/business","display_name":"Business","score":0.18837210536003113},{"id":"https://openalex.org/keywords/social-science","display_name":"Social science","score":0.09630987048149109},{"id":"https://openalex.org/keywords/developmental-psychology","display_name":"Developmental psychology","score":0.0842021107673645}],"concepts":[{"id":"https://openalex.org/C61863361","wikidata":"https://www.wikidata.org/wiki/Q470749","display_name":"Inter-rater reliability","level":3,"score":0.9097204208374023},{"id":"https://openalex.org/C2781316041","wikidata":"https://www.wikidata.org/wiki/Q1230584","display_name":"Diversity (politics)","level":2,"score":0.5886318683624268},{"id":"https://openalex.org/C108170787","wikidata":"https://www.wikidata.org/wiki/Q3951828","display_name":"Agency (philosophy)","level":2,"score":0.5701032280921936},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.5355631709098816},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.5023181438446045},{"id":"https://openalex.org/C138368954","wikidata":"https://www.wikidata.org/wiki/Q215028","display_name":"Peer review","level":2,"score":0.47959744930267334},{"id":"https://openalex.org/C162118730","wikidata":"https://www.wikidata.org/wiki/Q1128453","display_name":"Actuarial science","level":1,"score":0.4090949296951294},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.24185344576835632},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.1959158182144165},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.18837210536003113},{"id":"https://openalex.org/C36289849","wikidata":"https://www.wikidata.org/wiki/Q34749","display_name":"Social science","level":1,"score":0.09630987048149109},{"id":"https://openalex.org/C138496976","wikidata":"https://www.wikidata.org/wiki/Q175002","display_name":"Developmental psychology","level":1,"score":0.0842021107673645},{"id":"https://openalex.org/C83849319","wikidata":"https://www.wikidata.org/wiki/Q7295720","display_name":"Rating scale","level":2,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1162/qss_a_00207","is_oa":true,"landing_page_url":"https://doi.org/10.1162/qss_a_00207","pdf_url":"https://direct.mit.edu/qss/article-pdf/doi/10.1162/qss_a_00207/2052976/qss_a_00207.pdf","source":{"id":"https://openalex.org/S4210195326","display_name":"Quantitative Science Studies","issn_l":"2641-3337","issn":["2641-3337"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310315718","host_organization_name":"The MIT Press","host_organization_lineage":["https://openalex.org/P4310315718"],"host_organization_lineage_names":["The MIT Press"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Quantitative Science Studies","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:9ab689a355a44af484c75f8cfe2865bb","is_oa":true,"landing_page_url":"https://doaj.org/article/9ab689a355a44af484c75f8cfe2865bb","pdf_url":null,"source":{"id":"https://openalex.org/S112646816","display_name":"SHILAP Revista de lepidopterolog\u00eda","issn_l":"0300-5267","issn":["0300-5267","2340-4078"],"is_oa":true,"is_in_doaj":true,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Quantitative Science Studies, Vol 3, Iss 3 (2022)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1162/qss_a_00207","is_oa":true,"landing_page_url":"https://doi.org/10.1162/qss_a_00207","pdf_url":"https://direct.mit.edu/qss/article-pdf/doi/10.1162/qss_a_00207/2052976/qss_a_00207.pdf","source":{"id":"https://openalex.org/S4210195326","display_name":"Quantitative Science Studies","issn_l":"2641-3337","issn":["2641-3337"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310315718","host_organization_name":"The MIT Press","host_organization_lineage":["https://openalex.org/P4310315718"],"host_organization_lineage_names":["The MIT Press"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Quantitative Science Studies","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Partnerships for the goals","score":0.5299999713897705,"id":"https://metadata.un.org/sdg/17"}],"awards":[{"id":"https://openalex.org/G456099245","display_name":null,"funder_award_id":"17/SPR/5319","funder_id":"https://openalex.org/F4320320847","funder_display_name":"Science Foundation Ireland"}],"funders":[{"id":"https://openalex.org/F4320320847","display_name":"Science Foundation Ireland","ror":"https://ror.org/0271asj38"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4293080803.pdf","grobid_xml":"https://content.openalex.org/works/W4293080803.grobid-xml"},"referenced_works_count":66,"referenced_works":["https://openalex.org/W56018600","https://openalex.org/W88307301","https://openalex.org/W644992242","https://openalex.org/W1975676436","https://openalex.org/W1976219213","https://openalex.org/W1977862340","https://openalex.org/W1979116104","https://openalex.org/W1980158193","https://openalex.org/W1980898285","https://openalex.org/W1982586701","https://openalex.org/W1985045716","https://openalex.org/W2000444639","https://openalex.org/W2002502156","https://openalex.org/W2005950835","https://openalex.org/W2020465553","https://openalex.org/W2025381180","https://openalex.org/W2026241885","https://openalex.org/W2041604826","https://openalex.org/W2042269644","https://openalex.org/W2051213584","https://openalex.org/W2055124066","https://openalex.org/W2061653610","https://openalex.org/W2063743914","https://openalex.org/W2066752129","https://openalex.org/W2074626404","https://openalex.org/W2086059573","https://openalex.org/W2102073367","https://openalex.org/W2102709459","https://openalex.org/W2106224201","https://openalex.org/W2106469901","https://openalex.org/W2111275873","https://openalex.org/W2113026508","https://openalex.org/W2116450759","https://openalex.org/W2137481238","https://openalex.org/W2147830671","https://openalex.org/W2167919917","https://openalex.org/W2273766569","https://openalex.org/W2327037637","https://openalex.org/W2331018588","https://openalex.org/W2498221266","https://openalex.org/W2582743722","https://openalex.org/W2720291100","https://openalex.org/W2744367301","https://openalex.org/W2755257852","https://openalex.org/W2792200862","https://openalex.org/W2883965652","https://openalex.org/W2905544742","https://openalex.org/W2965156361","https://openalex.org/W2969551945","https://openalex.org/W2995185148","https://openalex.org/W3010278243","https://openalex.org/W3021792676","https://openalex.org/W3038952826","https://openalex.org/W3096926724","https://openalex.org/W3103531977","https://openalex.org/W3158913017","https://openalex.org/W3175713277","https://openalex.org/W4206964233","https://openalex.org/W4211096163","https://openalex.org/W4214814665","https://openalex.org/W4233482170","https://openalex.org/W4237327899","https://openalex.org/W4246024988","https://openalex.org/W4254168416","https://openalex.org/W4281757436","https://openalex.org/W6939566815"],"related_works":["https://openalex.org/W271875044","https://openalex.org/W4321371874","https://openalex.org/W3113237856","https://openalex.org/W2105374537","https://openalex.org/W2069853727","https://openalex.org/W3042168550","https://openalex.org/W2468427356","https://openalex.org/W2349480619","https://openalex.org/W2022240134","https://openalex.org/W1944348779"],"abstract_inverted_index":{"Abstract":[0],"One":[1],"of":[2,6,72,100,115,121,138,172,176],"the":[3,69],"main":[4],"critiques":[5],"academic":[7],"peer":[8,177],"review":[9,70,101,178],"is":[10,17,156],"that":[11,165],"interrater":[12],"reliability":[13],"(IRR)":[14],"among":[15,40],"reviewers":[16,41,84],"low.":[18],"We":[19],"examine":[20],"an":[21],"underinvestigated":[22],"factor":[23],"possibly":[24],"contributing":[25],"to":[26,38,43,48,79,93,104,158,180],"low":[27,94],"IRR:":[28],"reviewers\u2019":[29,166],"diversity":[30],"in":[31,50,75,86,124,143],"their":[32,51,90,125],"topic-criteria":[33],"mapping":[34],"(\u201cTC-mapping\u201d).":[35],"It":[36],"refers":[37],"differences":[39,91,123,142],"pertaining":[42],"which":[44],"topics":[45,58],"they":[46,55],"choose":[47],"emphasize":[49],"evaluations,":[52],"and":[53,88,112],"how":[54],"map":[56],"those":[57],"onto":[59],"various":[60],"evaluation":[61],"criteria.":[62],"In":[63],"this":[64],"paper":[65],"we":[66,118,131],"look":[67],"at":[68],"process":[71],"grant":[73],"proposals":[74],"one":[76],"funding":[77,107],"agency":[78,108],"ask:":[80],"How":[81],"much":[82],"do":[83,89],"differ":[85],"TC-mapping,":[87],"contribute":[92],"IRR?":[95],"Through":[96],"a":[97,105,113,128,135,147],"content":[98],"analysis":[99],"forms":[102],"submitted":[103],"national":[106],"(Science":[109],"Foundation":[110],"Ireland)":[111],"survey":[114],"its":[116],"reviewers,":[117],"find":[119],"evidence":[120],"interreviewer":[122],"TC-mapping.":[126],"Using":[127],"simulation":[129,160],"experiment":[130],"show":[132],"that,":[133],"under":[134],"wide":[136],"range":[137],"conditions,":[139],"even":[140],"strong":[141],"TC-mapping":[144],"have":[145],"only":[146],"negligible":[148],"impact":[149],"on":[150],"IRR.":[151,182],"Although":[152],"further":[153],"empirical":[154],"work":[155],"needed":[157],"corroborate":[159],"results,":[161],"these":[162],"tentatively":[163],"suggest":[164],"heterogeneous":[167],"TC-mappings":[168],"might":[169],"not":[170],"be":[171],"concern":[173],"for":[174],"designers":[175],"panels":[179],"safeguard":[181]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":2}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}