{"id":"https://openalex.org/W4391230125","doi":"https://doi.org/10.1007/s11192-023-04915-y","title":"How do referees integrate evaluation criteria into their overall judgment? Evidence from grant peer review","display_name":"How do referees integrate evaluation criteria into their overall judgment? Evidence from grant peer review","publication_year":2024,"publication_date":"2024-01-25","ids":{"openalex":"https://openalex.org/W4391230125","doi":"https://doi.org/10.1007/s11192-023-04915-y"},"language":"en","primary_location":{"id":"doi:10.1007/s11192-023-04915-y","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s11192-023-04915-y","pdf_url":"https://link.springer.com/content/pdf/10.1007/s11192-023-04915-y.pdf","source":{"id":"https://openalex.org/S148561398","display_name":"Scientometrics","issn_l":"0138-9130","issn":["0138-9130","1588-2861"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320108","host_organization_name":"Springer Nature (Netherlands)","host_organization_lineage":["https://openalex.org/P4310320108","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Nature (Netherlands)","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Scientometrics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://link.springer.com/content/pdf/10.1007/s11192-023-04915-y.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5085946061","display_name":"Sven E. Hug","orcid":"https://orcid.org/0000-0002-7624-9529"},"institutions":[{"id":"https://openalex.org/I202697423","display_name":"University of Zurich","ror":"https://ror.org/02crff812","country_code":"CH","type":"education","lineage":["https://openalex.org/I202697423"]}],"countries":["CH"],"is_corresponding":true,"raw_author_name":"Sven E. Hug","raw_affiliation_strings":["Department of Psychology, University of Zurich, Binzm\u00fchlestrasse 14/13, 8050, Zurich, Switzerland"],"affiliations":[{"raw_affiliation_string":"Department of Psychology, University of Zurich, Binzm\u00fchlestrasse 14/13, 8050, Zurich, Switzerland","institution_ids":["https://openalex.org/I202697423"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5085946061"],"corresponding_institution_ids":["https://openalex.org/I202697423"],"apc_list":{"value":2290,"currency":"EUR","value_usd":2890},"apc_paid":{"value":2290,"currency":"EUR","value_usd":2890},"fwci":6.224,"has_fulltext":true,"cited_by_count":13,"citation_normalized_percentile":{"value":0.96478873,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":100},"biblio":{"volume":"129","issue":"3","first_page":"1231","last_page":"1253"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10206","display_name":"Meta-analysis and systematic reviews","score":0.9887999892234802,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10206","display_name":"Meta-analysis and systematic reviews","score":0.9887999892234802,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10102","display_name":"scientometrics and bibliometrics research","score":0.9865000247955322,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10315","display_name":"Decision-Making and Behavioral Economics","score":0.9728999733924866,"subfield":{"id":"https://openalex.org/subfields/1800","display_name":"General Decision Sciences"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/heuristics","display_name":"Heuristics","score":0.7392683625221252},{"id":"https://openalex.org/keywords/normative","display_name":"Normative","score":0.6727780103683472},{"id":"https://openalex.org/keywords/style","display_name":"Style (visual arts)","score":0.617478609085083},{"id":"https://openalex.org/keywords/heuristic","display_name":"Heuristic","score":0.587681770324707},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.4970698654651642},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.48208171129226685},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.4744071662425995},{"id":"https://openalex.org/keywords/logistic-regression","display_name":"Logistic regression","score":0.46244603395462036},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.2603611648082733},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.2053111493587494}],"concepts":[{"id":"https://openalex.org/C127705205","wikidata":"https://www.wikidata.org/wiki/Q5748245","display_name":"Heuristics","level":2,"score":0.7392683625221252},{"id":"https://openalex.org/C44725695","wikidata":"https://www.wikidata.org/wiki/Q288156","display_name":"Normative","level":2,"score":0.6727780103683472},{"id":"https://openalex.org/C2776445246","wikidata":"https://www.wikidata.org/wiki/Q1792644","display_name":"Style (visual arts)","level":2,"score":0.617478609085083},{"id":"https://openalex.org/C173801870","wikidata":"https://www.wikidata.org/wiki/Q201413","display_name":"Heuristic","level":2,"score":0.587681770324707},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4970698654651642},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.48208171129226685},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.4744071662425995},{"id":"https://openalex.org/C151956035","wikidata":"https://www.wikidata.org/wiki/Q1132755","display_name":"Logistic regression","level":2,"score":0.46244603395462036},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2603611648082733},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2053111493587494},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1007/s11192-023-04915-y","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s11192-023-04915-y","pdf_url":"https://link.springer.com/content/pdf/10.1007/s11192-023-04915-y.pdf","source":{"id":"https://openalex.org/S148561398","display_name":"Scientometrics","issn_l":"0138-9130","issn":["0138-9130","1588-2861"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320108","host_organization_name":"Springer Nature (Netherlands)","host_organization_lineage":["https://openalex.org/P4310320108","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Nature (Netherlands)","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Scientometrics","raw_type":"journal-article"},{"id":"pmh:oai:RePEc:spr:scient:v:129:y:2024:i:3:d:10.1007_s11192-023-04915-y","is_oa":false,"landing_page_url":"http://link.springer.com/10.1007/s11192-023-04915-y","pdf_url":null,"source":{"id":"https://openalex.org/S4306401271","display_name":"RePEc: Research Papers in Economics","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I77793887","host_organization_name":"Federal Reserve Bank of St. Louis","host_organization_lineage":["https://openalex.org/I77793887"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"doi:10.1007/s11192-023-04915-y","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s11192-023-04915-y","pdf_url":"https://link.springer.com/content/pdf/10.1007/s11192-023-04915-y.pdf","source":{"id":"https://openalex.org/S148561398","display_name":"Scientometrics","issn_l":"0138-9130","issn":["0138-9130","1588-2861"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320108","host_organization_name":"Springer Nature (Netherlands)","host_organization_lineage":["https://openalex.org/P4310320108","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Nature (Netherlands)","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Scientometrics","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320323874","display_name":"Universit\u00e4t Z\u00fcrich","ror":"https://ror.org/02crff812"}],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4391230125.pdf"},"referenced_works_count":90,"referenced_works":["https://openalex.org/W154242586","https://openalex.org/W260994251","https://openalex.org/W772723537","https://openalex.org/W1482004794","https://openalex.org/W1504162107","https://openalex.org/W1792586875","https://openalex.org/W1845434773","https://openalex.org/W1849695254","https://openalex.org/W1864271787","https://openalex.org/W1964720683","https://openalex.org/W1972764464","https://openalex.org/W1995412971","https://openalex.org/W2003572022","https://openalex.org/W2005950835","https://openalex.org/W2025406302","https://openalex.org/W2030552375","https://openalex.org/W2042269644","https://openalex.org/W2055320743","https://openalex.org/W2059135017","https://openalex.org/W2060167209","https://openalex.org/W2060176298","https://openalex.org/W2060479619","https://openalex.org/W2075456177","https://openalex.org/W2094721029","https://openalex.org/W2096016260","https://openalex.org/W2096582410","https://openalex.org/W2102256472","https://openalex.org/W2114896740","https://openalex.org/W2121788424","https://openalex.org/W2140543255","https://openalex.org/W2142257341","https://openalex.org/W2150510315","https://openalex.org/W2152502221","https://openalex.org/W2155648416","https://openalex.org/W2157157156","https://openalex.org/W2159071801","https://openalex.org/W2181741437","https://openalex.org/W2211385352","https://openalex.org/W2242842357","https://openalex.org/W2273766569","https://openalex.org/W2325393906","https://openalex.org/W2358148633","https://openalex.org/W2413221400","https://openalex.org/W2498221266","https://openalex.org/W2514214557","https://openalex.org/W2533264585","https://openalex.org/W2582743722","https://openalex.org/W2658830562","https://openalex.org/W2744304370","https://openalex.org/W2755073967","https://openalex.org/W2763785586","https://openalex.org/W2769972173","https://openalex.org/W2788415229","https://openalex.org/W2844198059","https://openalex.org/W2883709920","https://openalex.org/W2909206408","https://openalex.org/W2941282355","https://openalex.org/W2951134316","https://openalex.org/W2964211782","https://openalex.org/W2965156361","https://openalex.org/W3033797387","https://openalex.org/W3035039854","https://openalex.org/W3045404545","https://openalex.org/W3090368714","https://openalex.org/W3099348159","https://openalex.org/W3103531977","https://openalex.org/W3118038935","https://openalex.org/W3118661838","https://openalex.org/W3118864605","https://openalex.org/W3175713277","https://openalex.org/W3183893045","https://openalex.org/W3201280195","https://openalex.org/W4200050117","https://openalex.org/W4205812422","https://openalex.org/W4210244353","https://openalex.org/W4210617664","https://openalex.org/W4214814665","https://openalex.org/W4221043148","https://openalex.org/W4223985076","https://openalex.org/W4240337989","https://openalex.org/W4251768930","https://openalex.org/W4253502614","https://openalex.org/W4283767871","https://openalex.org/W4292157289","https://openalex.org/W4293144432","https://openalex.org/W4367693914","https://openalex.org/W4385329495","https://openalex.org/W6601680532","https://openalex.org/W6677415673","https://openalex.org/W6890115044"],"related_works":["https://openalex.org/W2280422768","https://openalex.org/W3143197806","https://openalex.org/W2037802808","https://openalex.org/W3177062893","https://openalex.org/W3125143773","https://openalex.org/W2007032764","https://openalex.org/W803550684","https://openalex.org/W2483226803","https://openalex.org/W4352977312","https://openalex.org/W2041180560"],"abstract_inverted_index":{"Abstract":[0],"Little":[1],"is":[2],"known":[3],"whether":[4],"peer":[5],"reviewers":[6],"use":[7,42,71,190],"the":[8,16,38,67,76,80,84,87,92,141,151,163,167,173,180,195,204,212,230,235],"same":[9,81,88],"evaluation":[10,122,236],"criteria":[11,17,48,55,85,93,142,192,196],"and":[12,33,44,53,90,128,158,193,201,246],"how":[13,138,220],"they":[14],"integrate":[15,54,91,140,194],"into":[18,61,143],"their":[19,62,144],"overall":[20,63,145,153],"judgment.":[21,64,146],"This":[22],"study":[23,171],"therefore":[24,218],"proposed":[25],"two":[26,102],"assessment":[27,154],"styles":[28,103],"based":[29,94],"on":[30,49,95],"theoretical":[31,247],"perspectives":[32],"normative":[34],"positions.":[35],"According":[36,65],"to":[37,66,136,184],"case-by-case":[39,51],"style,":[40,69],"referees":[41,70,139,189,239],"many":[43,191],"different":[45],"criteria,":[46,78,82],"weight":[47,83],"a":[50,57,72,107,111,117,129],"basis,":[52],"in":[56,86,226,243],"complex,":[58],"non-mechanical":[59],"way":[60],"uniform":[68,174,181,231],"small":[73],"fraction":[74],"of":[75,121,169,238],"available":[77],"apply":[79],"way,":[89],"simple":[96],"rules":[97],"(i.e.,":[98],"fast-and-frugal":[99,164],"heuristics).":[100],"These":[101],"were":[104,134],"examined":[105],"using":[106,197],"unique":[108],"dataset":[109],"from":[110],"career":[112],"funding":[113],"scheme":[114],"that":[115,179],"contained":[116],"comparatively":[118],"large":[119],"number":[120],"criteria.":[123],"A":[124],"heuristic":[125],"(fast-and-frugal":[126],"trees)":[127],"complex":[130,198],"procedure":[131],"(logistic":[132],"regression)":[133],"employed":[135],"describe":[137,208],"The":[147],"logistic":[148],"regression":[149],"predicted":[150],"referees\u2019":[152,213,221],"with":[155],"high":[156],"accuracy":[157],"slightly":[159],"more":[160,244],"accurately":[161],"than":[162],"trees.":[165],"Overall,":[166],"results":[168],"this":[170],"support":[172],"style":[175,182,206,232],"but":[176],"also":[177],"indicate":[178],"needs":[183],"be":[185,224,241],"revised":[186,205],"as":[187],"follows:":[188],"rules.":[199],"However,":[200],"most":[202],"importantly,":[203],"could":[207],"most\u2014but":[209],"not":[210],"all\u2014of":[211],"judgments.":[214],"Future":[215],"studies":[216],"should":[217,240],"examine":[219],"judgments":[222],"can":[223],"characterized":[225],"those":[227],"cases":[228],"where":[229],"failed.":[233],"Moreover,":[234],"process":[237],"studied":[242],"empirical":[245],"detail.":[248]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":7},{"year":2024,"cited_by_count":3}],"updated_date":"2025-12-21T23:12:01.093139","created_date":"2025-10-10T00:00:00"}