{"id":"https://openalex.org/W1555073069","doi":"https://doi.org/10.1002/asi.23315","title":"Testing for the fairness and predictive validity of research funding decisions: A multilevel multiple imputation for missing data approach using ex\u2010ante and ex\u2010post peer evaluation data from the A ustrian science fund","display_name":"Testing for the fairness and predictive validity of research funding decisions: A multilevel multiple imputation for missing data approach using ex\u2010ante and ex\u2010post peer evaluation data from the A ustrian science fund","publication_year":2014,"publication_date":"2014-07-14","ids":{"openalex":"https://openalex.org/W1555073069","doi":"https://doi.org/10.1002/asi.23315","mag":"1555073069"},"language":"en","primary_location":{"id":"doi:10.1002/asi.23315","doi":"https://doi.org/10.1002/asi.23315","is_oa":true,"landing_page_url":"https://doi.org/10.1002/asi.23315","pdf_url":"https://onlinelibrary.wiley.com/doi/pdfdirect/10.1002/asi.23315","source":{"id":"https://openalex.org/S4210197613","display_name":"Journal of the Association for Information Science and Technology","issn_l":"2330-1635","issn":["2330-1635","2330-1643"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["anvur-class-a","anvur-scientific","ccf-b","cwts-core","dongbi-b","erih-plus","fnege-4","jufo-3","ki-jl-1","norway-2","poland-140","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320595","host_organization_name":"Wiley","host_organization_lineage":["https://openalex.org/P4310320595"],"host_organization_lineage_names":["Wiley"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of the Association for Information Science and Technology","raw_type":"journal-article"},"volume":null,"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://onlinelibrary.wiley.com/doi/pdfdirect/10.1002/asi.23315","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5028193565","display_name":"R\u00fcdiger Mutz","orcid":"https://orcid.org/0000-0003-3345-6090","observed_orcids":["https://orcid.org/0000-0003-3345-6090"]},"institutions":[{"id":"https://openalex.org/I35440088","display_name":"ETH Zurich","ror":"https://ror.org/05a28rw58","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I35440088"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"R\u00fcdiger Mutz","raw_affiliation_strings":["Professorship for Social Psychology and Research on Higher Education, ETH Zurich; Muehlegasse 21, 8001 Zurich Switzerland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Professorship for Social Psychology and Research on Higher Education, ETH Zurich; Muehlegasse 21, 8001 Zurich Switzerland","institution_ids":["https://openalex.org/I35440088"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027052270","display_name":"Lutz Bornmann","orcid":"https://orcid.org/0000-0003-0810-7091","observed_orcids":["https://orcid.org/0000-0003-0810-7091"]},"institutions":[{"id":"https://openalex.org/I149899117","display_name":"Max Planck Society","ror":"https://ror.org/01hhn8329","country_code":"DE","type":"nonprofit","lineage":["https://openalex.org/I149899117"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Lutz Bornmann","raw_affiliation_strings":["Administrative Headquarters of the Max Planck Society  Hofgartenstrasse 8 D\u201080539 Munich Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Administrative Headquarters of the Max Planck Society  Hofgartenstrasse 8 D\u201080539 Munich Germany","institution_ids":["https://openalex.org/I149899117"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5038583366","display_name":"Hans\u2010Dieter Daniel","orcid":"https://orcid.org/0000-0001-7925-1846","observed_orcids":["https://orcid.org/0000-0001-7925-1846"]},"institutions":[{"id":"https://openalex.org/I202697423","display_name":"University of Zurich","ror":"https://ror.org/02crff812","country_code":"CH","type":"education","lineage":["https://openalex.org/I202697423"]},{"id":"https://openalex.org/I35440088","display_name":"ETH Zurich","ror":"https://ror.org/05a28rw58","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I35440088"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Hans\u2010Dieter Daniel","raw_affiliation_strings":["Evaluation Office University of Zurich Muehlegasse 21 8001 Zurich Switzerland","Professorship for Social Psychology and Research on Higher Education ETH Zurich Muehlegasse 21 8001 Zurich Switzerland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Evaluation Office University of Zurich Muehlegasse 21 8001 Zurich Switzerland","institution_ids":["https://openalex.org/I202697423"]},{"raw_affiliation_string":"Professorship for Social Psychology and Research on Higher Education ETH Zurich Muehlegasse 21 8001 Zurich Switzerland","institution_ids":["https://openalex.org/I35440088"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":{"value":3310,"currency":"USD","value_usd":3310},"apc_paid":{"value":3310,"currency":"USD","value_usd":3310},"fwci":1.8226,"has_fulltext":true,"cited_by_count":21,"citation_normalized_percentile":{"value":0.87239994,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":88,"max":98},"biblio":{"volume":"66","issue":"11","first_page":"2321","last_page":"2339"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10102","display_name":"scientometrics and bibliometrics research","score":0.3246999979019165,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10102","display_name":"scientometrics and bibliometrics research","score":0.3246999979019165,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T14185","display_name":"Research, Science, and Academia","score":0.2502000033855438,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13284","display_name":"Psychology Research and Bibliometrics","score":0.19699999690055847,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/research-funding-decisions","display_name":"research funding decisions","score":0.9279999732971191},{"id":"https://openalex.org/keywords/peer-review","display_name":"peer review","score":0.8629999756813049},{"id":"https://openalex.org/keywords/fairness","display_name":"fairness","score":0.39899998903274536},{"id":"https://openalex.org/keywords/predictive-validity","display_name":"predictive validity","score":0.9959999918937683},{"id":"https://openalex.org/keywords/verification-bias","display_name":"verification bias","score":0.9660000205039978},{"id":"https://openalex.org/keywords/missing-data-imputation","display_name":"missing data imputation","score":0.9259999990463257},{"id":"https://openalex.org/keywords/multilevel-modeling","display_name":"multilevel modeling","score":0.8980000019073486},{"id":"https://openalex.org/keywords/austrian-science-fund","display_name":"Austrian Science Fund","score":0.8450000286102295},{"id":"https://openalex.org/keywords/grant-applications","display_name":"grant applications","score":0.6510000228881836},{"id":"https://openalex.org/keywords/gender-bias","display_name":"gender bias","score":0.8410000205039978},{"id":"https://openalex.org/keywords/disciplinary-bias","display_name":"disciplinary bias","score":0.6359999775886536}],"concepts":[{"id":"https://openalex.org/C9357733","wikidata":"https://www.wikidata.org/wiki/Q6878417","display_name":"Missing data","level":2,"score":0.8186181783676147},{"id":"https://openalex.org/C58041806","wikidata":"https://www.wikidata.org/wiki/Q1660484","display_name":"Imputation (statistics)","level":3,"score":0.817084550857544},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6223423480987549},{"id":"https://openalex.org/C2778738651","wikidata":"https://www.wikidata.org/wiki/Q16546687","display_name":"Novelty","level":2,"score":0.5927338004112244},{"id":"https://openalex.org/C174106493","wikidata":"https://www.wikidata.org/wiki/Q1057880","display_name":"External validity","level":2,"score":0.47227367758750916},{"id":"https://openalex.org/C122251271","wikidata":"https://www.wikidata.org/wiki/Q940039","display_name":"Ex-ante","level":2,"score":0.4307713806629181},{"id":"https://openalex.org/C20685875","wikidata":"https://www.wikidata.org/wiki/Q7239678","display_name":"Predictive validity","level":2,"score":0.41577446460723877},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.38492685556411743},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.317976176738739},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.27363109588623047},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.1462710201740265},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.13606050610542297},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.1217677891254425},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.10511523485183716},{"id":"https://openalex.org/C139719470","wikidata":"https://www.wikidata.org/wiki/Q39680","display_name":"Macroeconomics","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1002/asi.23315","doi":"https://doi.org/10.1002/asi.23315","is_oa":true,"landing_page_url":"https://doi.org/10.1002/asi.23315","pdf_url":"https://onlinelibrary.wiley.com/doi/pdfdirect/10.1002/asi.23315","source":{"id":"https://openalex.org/S4210197613","display_name":"Journal of the Association for Information Science and Technology","issn_l":"2330-1635","issn":["2330-1635","2330-1643"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["anvur-class-a","anvur-scientific","ccf-b","cwts-core","dongbi-b","erih-plus","fnege-4","jufo-3","ki-jl-1","norway-2","poland-140","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320595","host_organization_name":"Wiley","host_organization_lineage":["https://openalex.org/P4310320595"],"host_organization_lineage_names":["Wiley"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of the Association for Information Science and Technology","raw_type":"journal-article"},{"id":"pmh:oai:RePEc:bla:jinfst:v:66:y:2015:i:11:p:2321-2339","doi":null,"is_oa":false,"landing_page_url":"http://hdl.handle.net/10.1002/asi.23315","pdf_url":null,"source":{"id":"https://openalex.org/S4306401271","display_name":"RePEc: Research Papers in Economics","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I77793887","host_organization_name":"Federal Reserve Bank of St. Louis","host_organization_lineage":["https://openalex.org/I77793887"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},{"id":"pmh:oai:zenodo.org:50822","doi":null,"is_oa":true,"landing_page_url":"https://www.openaccessrepository.it/record/50822","pdf_url":null,"source":{"id":"https://openalex.org/S4306402478","display_name":"INFM-OAR (INFN Catania)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I4210116497","host_organization_name":"Istituto Nazionale di Fisica Nucleare, Sezione di Catania","host_organization_lineage":["https://openalex.org/I4210116497"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":{"id":"doi:10.1002/asi.23315","doi":"https://doi.org/10.1002/asi.23315","is_oa":true,"landing_page_url":"https://doi.org/10.1002/asi.23315","pdf_url":"https://onlinelibrary.wiley.com/doi/pdfdirect/10.1002/asi.23315","source":{"id":"https://openalex.org/S4210197613","display_name":"Journal of the Association for Information Science and Technology","issn_l":"2330-1635","issn":["2330-1635","2330-1643"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["anvur-class-a","anvur-scientific","ccf-b","cwts-core","dongbi-b","erih-plus","fnege-4","jufo-3","ki-jl-1","norway-2","poland-140","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320595","host_organization_name":"Wiley","host_organization_lineage":["https://openalex.org/P4310320595"],"host_organization_lineage_names":["Wiley"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of the Association for Information Science and Technology","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/10","display_name":"Reduced inequalities","score":0.4000000059604645}],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320321181","display_name":"Austrian Science Fund","ror":"https://ror.org/013tf3c58"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W1555073069.pdf","grobid_xml":"https://content.openalex.org/works/W1555073069.grobid-xml"},"referenced_works_count":137,"referenced_works":["https://openalex.org/W114148772","https://openalex.org/W136966255","https://openalex.org/W355727244","https://openalex.org/W612972800","https://openalex.org/W1487018400","https://openalex.org/W1495176512","https://openalex.org/W1529643479","https://openalex.org/W1550448043","https://openalex.org/W1562168738","https://openalex.org/W1572806907","https://openalex.org/W1581096047","https://openalex.org/W1581398571","https://openalex.org/W1610612296","https://openalex.org/W1887831276","https://openalex.org/W1894962167","https://openalex.org/W1895540621","https://openalex.org/W1919216911","https://openalex.org/W1937603598","https://openalex.org/W1969240051","https://openalex.org/W1970469992","https://openalex.org/W1971452447","https://openalex.org/W1972025636","https://openalex.org/W1972325682","https://openalex.org/W1972541135","https://openalex.org/W1973151127","https://openalex.org/W1973490526","https://openalex.org/W1977862340","https://openalex.org/W1978883524","https://openalex.org/W1981678461","https://openalex.org/W1982930336","https://openalex.org/W1989829179","https://openalex.org/W1990544598","https://openalex.org/W1992043444","https://openalex.org/W1992531794","https://openalex.org/W1994028791","https://openalex.org/W1996960755","https://openalex.org/W1997241682","https://openalex.org/W1999086396","https://openalex.org/W2004913798","https://openalex.org/W2005928344","https://openalex.org/W2005950835","https://openalex.org/W2006828034","https://openalex.org/W2010940417","https://openalex.org/W2012152988","https://openalex.org/W2013662625","https://openalex.org/W2014134784","https://openalex.org/W2018192258","https://openalex.org/W2020034787","https://openalex.org/W2021506523","https://openalex.org/W2022950201","https://openalex.org/W2024964989","https://openalex.org/W2025358672","https://openalex.org/W2026162142","https://openalex.org/W2026286904","https://openalex.org/W2027554232","https://openalex.org/W2030294407","https://openalex.org/W2030320984","https://openalex.org/W2031366728","https://openalex.org/W2033886860","https://openalex.org/W2037632869","https://openalex.org/W2039065426","https://openalex.org/W2042269644","https://openalex.org/W2044758663","https://openalex.org/W2051213584","https://openalex.org/W2051383271","https://openalex.org/W2052562402","https://openalex.org/W2054181045","https://openalex.org/W2054407767","https://openalex.org/W2057777484","https://openalex.org/W2059131090","https://openalex.org/W2061980801","https://openalex.org/W2062077255","https://openalex.org/W2064628008","https://openalex.org/W2065974896","https://openalex.org/W2066297573","https://openalex.org/W2068267890","https://openalex.org/W2069196044","https://openalex.org/W2073587963","https://openalex.org/W2074978652","https://openalex.org/W2075161400","https://openalex.org/W2082957927","https://openalex.org/W2083273322","https://openalex.org/W2086406824","https://openalex.org/W2087826418","https://openalex.org/W2089976450","https://openalex.org/W2091174313","https://openalex.org/W2093601894","https://openalex.org/W2095116045","https://openalex.org/W2096394418","https://openalex.org/W2102932588","https://openalex.org/W2108148387","https://openalex.org/W2109574134","https://openalex.org/W2111022718","https://openalex.org/W2111547326","https://openalex.org/W2111938618","https://openalex.org/W2114123404","https://openalex.org/W2115098571","https://openalex.org/W2119252543","https://openalex.org/W2119292158","https://openalex.org/W2119393870","https://openalex.org/W2125027669","https://openalex.org/W2128193301","https://openalex.org/W2129513604","https://openalex.org/W2132963080","https://openalex.org/W2133494987","https://openalex.org/W2135551891","https://openalex.org/W2137484401","https://openalex.org/W2138115784","https://openalex.org/W2138269352","https://openalex.org/W2138414141","https://openalex.org/W2143317665","https://openalex.org/W2147830671","https://openalex.org/W2149511379","https://openalex.org/W2150583095","https://openalex.org/W2153142912","https://openalex.org/W2157275878","https://openalex.org/W2162196730","https://openalex.org/W2231840137","https://openalex.org/W2236322269","https://openalex.org/W2291133860","https://openalex.org/W2312978426","https://openalex.org/W2322909744","https://openalex.org/W2332991782","https://openalex.org/W2480680997","https://openalex.org/W2502759836","https://openalex.org/W2512018981","https://openalex.org/W2514055248","https://openalex.org/W2552830112","https://openalex.org/W2593724198","https://openalex.org/W2600784572","https://openalex.org/W2782623783","https://openalex.org/W2890373521","https://openalex.org/W2900595267","https://openalex.org/W4211254511","https://openalex.org/W4236891926","https://openalex.org/W4240233136","https://openalex.org/W4241561347"],"related_works":["https://openalex.org/W1973151127","https://openalex.org/W2072020930","https://openalex.org/W6969021808","https://openalex.org/W2744367301","https://openalex.org/W3164476488","https://openalex.org/W2355139284","https://openalex.org/W1980669377","https://openalex.org/W4414846632","https://openalex.org/W2963896435","https://openalex.org/W2051213584"],"abstract_inverted_index":{"It":[0],"is":[1,113],"essential":[2],"for":[3,56,75,121,173,186],"research":[4,46,119],"funding":[5,184],"organizations":[6],"to":[7,31,38,71,97,135,171,180],"ensure":[8],"both":[9,187],"the":[10,15,32,57,60,76,84,110,117,124,136,156],"validity":[11,86,162,188],"and":[12,100,145,163,189,195],"fairness":[13,164,190],"of":[14,24,59,87,115,151,158,161,193],"grant":[16,28,61],"approval":[17],"procedure.":[18],"The":[19,149],"ex\u2010ante":[20],"peer":[21,50,182],"evaluation":[22,51],"(EXANTE)":[23],"N":[25],"=":[26,92,106],"8,496":[27],"applications":[29,62],"submitted":[30],"Austrian":[33],"Science":[34],"Fund":[35],"from":[36],"1999":[37],"2009":[39],"was":[40,53,69,89,104,139],"statistically":[41],"analyzed.":[42],"For":[43],"1,689":[44],"funded":[45],"projects":[47],"an":[48],"ex\u2010post":[49],"(EXPOST)":[52],"also":[54],"available;":[55],"rest":[58],"a":[63,166],"multilevel":[64,178],"missing":[65,167],"data":[66,168],"imputation":[67,102,169],"approach":[68,170],"used":[70],"consider":[72],"verification":[73,98,174],"bias":[74,143,175],"first":[77],"time":[78],"in":[79,191],"peer\u2010review":[80],"research.":[81],"Without":[82],"imputation,":[83],"predictive":[85],"EXANTE":[88,125],"low":[90],"(r":[91],".26)":[93],"but":[94],"underestimated":[95],"due":[96],"bias,":[99],"with":[101,165],"it":[103],"r":[105],".49.":[107],"That":[108],"is,":[109,154],"decision\u2010making":[111],"procedure":[112],"capable":[114],"selecting":[116],"best":[118],"proposals":[120],"funding.":[122],"In":[123],"there":[126,138],"were":[127],"several":[128],"potential":[129,194],"biases":[130],"(e.g.,":[131],"gender).":[132],"With":[133],"respect":[134],"EXPOST":[137],"only":[140],"one":[141],"real":[142,196],"(discipline\u2010specific":[144],"year\u2010specific":[146],"differential":[147],"prediction).":[148],"novelty":[150],"this":[152],"contribution":[153],"first,":[155],"combining":[157],"theoretical":[159],"concepts":[160],"correct":[172],"and,":[176],"second,":[177],"modeling":[179],"test":[181],"review\u2010based":[183],"decisions":[185],"terms":[192],"biases.":[197]},"counts_by_year":[{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":2},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":3},{"year":2019,"cited_by_count":4},{"year":2018,"cited_by_count":1},{"year":2017,"cited_by_count":1},{"year":2016,"cited_by_count":4},{"year":2015,"cited_by_count":1}],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2016-06-24T00:00:00"}