{"id":"https://openalex.org/W4404602011","doi":"https://doi.org/10.1017/jdm.2024.24","title":"Evaluating science: A comparison of human and AI reviewers","display_name":"Evaluating science: A comparison of human and AI reviewers","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4404602011","doi":"https://doi.org/10.1017/jdm.2024.24"},"language":"en","primary_location":{"id":"doi:10.1017/jdm.2024.24","doi":"https://doi.org/10.1017/jdm.2024.24","is_oa":true,"landing_page_url":"https://doi.org/10.1017/jdm.2024.24","pdf_url":"https://www.cambridge.org/core/services/aop-cambridge-core/content/view/6F69123851472B4A72DEC0BD08C13AA6/S193029752400024Xa.pdf/div-class-title-evaluating-science-a-comparison-of-human-and-ai-reviewers-div.pdf","source":{"id":"https://openalex.org/S179575189","display_name":"Judgment and Decision Making","issn_l":"1930-2975","issn":["1930-2975"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["abdc-a","anvur-class-a","anvur-scientific","cwts-core","doaj","dongbi-b","erih-plus","fnege-3","jufo-1","ki-jl-1","norway-1","poland-100","russia-white-list-2","vabb-shw"],"host_organization":"https://openalex.org/P4310311721","host_organization_name":"Cambridge University Press","host_organization_lineage":["https://openalex.org/P4310311721","https://openalex.org/P4310311702"],"host_organization_lineage_names":["Cambridge University Press","University of Cambridge"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Judgment and Decision Making","raw_type":"journal-article"},"volume":null,"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.cambridge.org/core/services/aop-cambridge-core/content/view/6F69123851472B4A72DEC0BD08C13AA6/S193029752400024Xa.pdf/div-class-title-evaluating-science-a-comparison-of-human-and-ai-reviewers-div.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5114744188","display_name":"Anna Shcherbiak","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I102248843","display_name":"Vienna University of Economics and Business","ror":"https://ror.org/03yn8s215","country_code":"AT","type":"education","lineage":["https://openalex.org/I102248843"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Anna Shcherbiak","raw_affiliation_strings":["Institute for Cognition and Behavior, WU Vienna University of Economics and Business, Vienna, Austria"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute for Cognition and Behavior, WU Vienna University of Economics and Business, Vienna, Austria","institution_ids":["https://openalex.org/I102248843"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054312664","display_name":"Hooman Habibnia","orcid":"https://orcid.org/0009-0000-3915-2624","observed_orcids":["https://orcid.org/0009-0000-3915-2624"]},"institutions":[{"id":"https://openalex.org/I102248843","display_name":"Vienna University of Economics and Business","ror":"https://ror.org/03yn8s215","country_code":"AT","type":"education","lineage":["https://openalex.org/I102248843"]}],"countries":["AT"],"is_corresponding":true,"raw_author_name":"Hooman Habibnia","raw_affiliation_strings":["Institute for Cognition and Behavior, WU Vienna University of Economics and Business, Vienna, Austria"],"raw_orcid":"https://orcid.org/0009-0000-3915-2624","affiliations":[{"raw_affiliation_string":"Institute for Cognition and Behavior, WU Vienna University of Economics and Business, Vienna, Austria","institution_ids":["https://openalex.org/I102248843"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075292368","display_name":"ROBERT M. BOHM","orcid":"https://orcid.org/0000-0001-6806-0374","observed_orcids":["https://orcid.org/0000-0001-6806-0374"]},"institutions":[{"id":"https://openalex.org/I124055696","display_name":"University of Copenhagen","ror":"https://ror.org/035b05819","country_code":"DK","type":"education","lineage":["https://openalex.org/I124055696"]},{"id":"https://openalex.org/I129774422","display_name":"University of Vienna","ror":"https://ror.org/03prydq77","country_code":"AT","type":"education","lineage":["https://openalex.org/I129774422"]}],"countries":["AT","DK"],"is_corresponding":false,"raw_author_name":"Robert B\u00f6hm","raw_affiliation_strings":["Department of Psychology and Copenhagen Center for Social Data Science (SODAS), University of Copenhagen, Copenhagen, Denmark","Faculty of Psychology, University of Vienna, Vienna, Austria","K\u00f8benhavns Universitet, Faculty of Social Sciences, Department of Psychology, Department of Psychology, DK"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Psychology and Copenhagen Center for Social Data Science (SODAS), University of Copenhagen, Copenhagen, Denmark","institution_ids":["https://openalex.org/I124055696"]},{"raw_affiliation_string":"Faculty of Psychology, University of Vienna, Vienna, Austria","institution_ids":["https://openalex.org/I129774422"]},{"raw_affiliation_string":"K\u00f8benhavns Universitet, Faculty of Social Sciences, Department of Psychology, Department of Psychology, DK","institution_ids":["https://openalex.org/I124055696"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5024420668","display_name":"Susann Fiedler","orcid":"https://orcid.org/0000-0001-9337-2142","observed_orcids":["https://orcid.org/0000-0001-9337-2142"]},"institutions":[{"id":"https://openalex.org/I102248843","display_name":"Vienna University of Economics and Business","ror":"https://ror.org/03yn8s215","country_code":"AT","type":"education","lineage":["https://openalex.org/I102248843"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Susann Fiedler","raw_affiliation_strings":["Institute for Cognition and Behavior, WU Vienna University of Economics and Business, Vienna, Austria"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute for Cognition and Behavior, WU Vienna University of Economics and Business, Vienna, Austria","institution_ids":["https://openalex.org/I102248843"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5054312664"],"corresponding_institution_ids":["https://openalex.org/I102248843"],"apc_list":{"value":3450,"currency":"USD","value_usd":3450},"apc_paid":{"value":3450,"currency":"USD","value_usd":3450},"fwci":19.3395,"has_fulltext":true,"cited_by_count":13,"citation_normalized_percentile":{"value":0.99249147,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":99},"biblio":{"volume":"19","issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12648","display_name":"Academic Writing and Publishing","score":0.2249000072479248,"subfield":{"id":"https://openalex.org/subfields/1207","display_name":"History and Philosophy of Science"},"field":{"id":"https://openalex.org/fields/12","display_name":"Arts and Humanities"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T12648","display_name":"Academic Writing and Publishing","score":0.2249000072479248,"subfield":{"id":"https://openalex.org/subfields/1207","display_name":"History and Philosophy of Science"},"field":{"id":"https://openalex.org/fields/12","display_name":"Arts and Humanities"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13607","display_name":"Academic Publishing and Open Access","score":0.1306000053882599,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13284","display_name":"Psychology Research and Bibliometrics","score":0.10130000114440918,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/ai-generated-text-detection","display_name":"AI-generated text detection","score":0.6890000104904175},{"id":"https://openalex.org/keywords/scientific-peer-review","display_name":"scientific peer review","score":0.828000009059906},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.8550000190734863},{"id":"https://openalex.org/keywords/gpt-4","display_name":"GPT-4","score":0.9950000047683716},{"id":"https://openalex.org/keywords/gptzero","display_name":"GPTZero","score":0.9760000109672546},{"id":"https://openalex.org/keywords/research-quality-assessment","display_name":"research quality assessment","score":0.7149999737739563},{"id":"https://openalex.org/keywords/authorship-attribution","display_name":"authorship attribution","score":0.5410000085830688}],"concepts":[{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.4732958674430847},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3764910399913788},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3523654341697693}],"mesh":[],"locations_count":5,"locations":[{"id":"doi:10.1017/jdm.2024.24","doi":"https://doi.org/10.1017/jdm.2024.24","is_oa":true,"landing_page_url":"https://doi.org/10.1017/jdm.2024.24","pdf_url":"https://www.cambridge.org/core/services/aop-cambridge-core/content/view/6F69123851472B4A72DEC0BD08C13AA6/S193029752400024Xa.pdf/div-class-title-evaluating-science-a-comparison-of-human-and-ai-reviewers-div.pdf","source":{"id":"https://openalex.org/S179575189","display_name":"Judgment and Decision Making","issn_l":"1930-2975","issn":["1930-2975"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["abdc-a","anvur-class-a","anvur-scientific","cwts-core","doaj","dongbi-b","erih-plus","fnege-3","jufo-1","ki-jl-1","norway-1","poland-100","russia-white-list-2","vabb-shw"],"host_organization":"https://openalex.org/P4310311721","host_organization_name":"Cambridge University Press","host_organization_lineage":["https://openalex.org/P4310311721","https://openalex.org/P4310311702"],"host_organization_lineage_names":["Cambridge University Press","University of Cambridge"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Judgment and Decision Making","raw_type":"journal-article"},{"id":"pmh:oai:pure.atira.dk:openaire/800f93ab-d215-43df-96b1-aed28ae686a9","doi":null,"is_oa":true,"landing_page_url":"https://researchprofiles.ku.dk/da/publications/800f93ab-d215-43df-96b1-aed28ae686a9","pdf_url":null,"source":{"id":"https://openalex.org/S4306401983","display_name":"Research at the University of Copenhagen (University of Copenhagen)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I124055696","host_organization_name":"University of Copenhagen","host_organization_lineage":["https://openalex.org/I124055696"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Shcherbiak, A, Habibnia, H, B\u00f6hm, R & Fiedler, S 2024, 'Evaluating science : A comparison of human and AI reviewers', Judgment and Decision Making, vol. 19, e21. https://doi.org/10.1017/jdm.2024.24","raw_type":"info:eu-repo/semantics/article"},{"id":"pmh:oai:RePEc:cup:judgdm:v:19:y:2024:i::p:-_21","doi":null,"is_oa":false,"landing_page_url":"https://www.cambridge.org/core/product/identifier/S193029752400024X/type/journal_article","pdf_url":null,"source":{"id":"https://openalex.org/S4306401271","display_name":"RePEc: Research Papers in Economics","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I77793887","host_organization_name":"Federal Reserve Bank of St. Louis","host_organization_lineage":["https://openalex.org/I77793887"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},{"id":"pmh:oai:doaj.org/article:77b7e2b6ba6046d9915c704b5d20d1be","doi":null,"is_oa":true,"landing_page_url":"https://doaj.org/article/77b7e2b6ba6046d9915c704b5d20d1be","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Judgment and Decision Making, Vol 19 (2024)","raw_type":"article"},{"id":"pmh:oai:share.osf.io:39126f0b-52f0-4054-9948-9dc9a238e75b","doi":null,"is_oa":true,"landing_page_url":"https://osf.io/nje23","pdf_url":null,"source":{"id":"https://openalex.org/S4306401127","display_name":"OSF Preprints (OSF Preprints)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2799848540","host_organization_name":"Center for Open Science","host_organization_lineage":["https://openalex.org/I2799848540"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Project"}],"best_oa_location":{"id":"doi:10.1017/jdm.2024.24","doi":"https://doi.org/10.1017/jdm.2024.24","is_oa":true,"landing_page_url":"https://doi.org/10.1017/jdm.2024.24","pdf_url":"https://www.cambridge.org/core/services/aop-cambridge-core/content/view/6F69123851472B4A72DEC0BD08C13AA6/S193029752400024Xa.pdf/div-class-title-evaluating-science-a-comparison-of-human-and-ai-reviewers-div.pdf","source":{"id":"https://openalex.org/S179575189","display_name":"Judgment and Decision Making","issn_l":"1930-2975","issn":["1930-2975"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["abdc-a","anvur-class-a","anvur-scientific","cwts-core","doaj","dongbi-b","erih-plus","fnege-3","jufo-1","ki-jl-1","norway-1","poland-100","russia-white-list-2","vabb-shw"],"host_organization":"https://openalex.org/P4310311721","host_organization_name":"Cambridge University Press","host_organization_lineage":["https://openalex.org/P4310311721","https://openalex.org/P4310311702"],"host_organization_lineage_names":["Cambridge University Press","University of Cambridge"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Judgment and Decision Making","raw_type":"journal-article"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4404602011.pdf","grobid_xml":"https://content.openalex.org/works/W4404602011.grobid-xml"},"referenced_works_count":30,"referenced_works":["https://openalex.org/W2005950835","https://openalex.org/W2276801815","https://openalex.org/W2551122076","https://openalex.org/W2592989848","https://openalex.org/W2726944738","https://openalex.org/W2917732241","https://openalex.org/W2980420460","https://openalex.org/W3099052861","https://openalex.org/W3214193544","https://openalex.org/W4239019441","https://openalex.org/W4243100767","https://openalex.org/W4282933389","https://openalex.org/W4282960557","https://openalex.org/W4285795866","https://openalex.org/W4313680034","https://openalex.org/W4317553041","https://openalex.org/W4318239126","https://openalex.org/W4324129401","https://openalex.org/W4327810158","https://openalex.org/W4360620450","https://openalex.org/W4367051110","https://openalex.org/W4376866708","https://openalex.org/W4379258778","https://openalex.org/W4388705463","https://openalex.org/W4388834645","https://openalex.org/W4392781833","https://openalex.org/W4396784687","https://openalex.org/W4400734098","https://openalex.org/W4414490687","https://openalex.org/W7133190912"],"related_works":["https://openalex.org/W7161948168","https://openalex.org/W7223332129","https://openalex.org/W7207648267","https://openalex.org/W7164148670","https://openalex.org/W4400105732","https://openalex.org/W7125838419","https://openalex.org/W7154601331","https://openalex.org/W4412736320","https://openalex.org/W7155065644","https://openalex.org/W4417497087"],"abstract_inverted_index":{"Abstract":[0],"Scientists":[1],"have":[2],"started":[3],"to":[4,29,39,68,166],"explore":[5],"whether":[6,32],"novel":[7],"artificial":[8],"intelligence":[9],"(AI)":[10],"tools":[11,174],"based":[12],"on":[13,49,83],"large":[14],"language":[15],"models,":[16],"such":[17],"as":[18],"GPT-4,":[19],"could":[20,152],"support":[21],"the":[22,55,64,144,167,176],"scientific":[23,80,158,177],"peer":[24,178],"review":[25,179],"process.":[26,180],"We":[27,71],"sought":[28],"understand":[30],"(i)":[31],"AI":[33,59,95,134,151,173],"versus":[34,60],"human":[35,61,98,104,140],"reviewers":[36,62,105,135,141],"are":[37,162],"able":[38],"distinguish":[40],"between":[41,126],"made-up":[42],"AI-generated":[43],"and":[44,52,86,106,129,170],"human-written":[45,85],"conference":[46],"abstracts":[47,89],"reporting":[48],"actual":[50],"research,":[51],"(ii)":[53],"how":[54],"quality":[56,119],"assessments":[57],"by":[58,94],"of":[63,172],"reported":[65],"research":[66],"correspond":[67],"each":[69],"other.":[70],"conducted":[72],"a":[73,78,154],"large-scale":[74],"field":[75],"experiment":[76],"during":[77,175],"medium-sized":[79],"conference,":[81],"relying":[82],"305":[84],"20":[87],"AI-written":[88],"that":[90,103,150],"were":[91,108,136],"reviewed":[92],"either":[93],"or":[96],"217":[97],"reviewers.":[99],"The":[100,160],"results":[101,161],"show":[102],"GPTZero":[107],"better":[109],"in":[110,142],"discerning":[111],"(AI":[112],"vs.":[113],"human)":[114],"authorship":[115],"than":[116],"GPT-4.":[117],"Regarding":[118],"assessments,":[120],"there":[121],"was":[122],"rather":[123],"low":[124],"agreement":[125],"both":[127],"human\u2013human":[128],"human\u2013AI":[130],"reviewer":[131],"pairs,":[132],"but":[133],"more":[137],"aligned":[138],"with":[139,164],"classifying":[143],"very":[145],"best":[146],"abstracts.":[147,159],"This":[148],"indicates":[149],"become":[153],"prescreening":[155],"tool":[156],"for":[157],"discussed":[163],"regard":[165],"future":[168],"development":[169],"use":[171]},"counts_by_year":[{"year":2026,"cited_by_count":9},{"year":2025,"cited_by_count":4}],"updated_date":"2026-10-16T09:59:22.235173","created_date":"2024-11-22T00:00:00"}