{"id":"https://openalex.org/W4395467630","doi":"https://doi.org/10.1007/s11023-024-09657-1","title":"Towards a Benchmark for Scientific Understanding in Humans and Machines","display_name":"Towards a Benchmark for Scientific Understanding in Humans and Machines","publication_year":2024,"publication_date":"2024-04-25","ids":{"openalex":"https://openalex.org/W4395467630","doi":"https://doi.org/10.1007/s11023-024-09657-1"},"language":"en","primary_location":{"id":"doi:10.1007/s11023-024-09657-1","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s11023-024-09657-1","pdf_url":"https://link.springer.com/content/pdf/10.1007/s11023-024-09657-1.pdf","source":{"id":"https://openalex.org/S160466889","display_name":"Minds and Machines","issn_l":"0924-6495","issn":["0924-6495","1572-8641"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Minds and Machines","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://link.springer.com/content/pdf/10.1007/s11023-024-09657-1.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5072006436","display_name":"Kristian Gonz\u00e1lez Barman","orcid":"https://orcid.org/0000-0001-7277-7351"},"institutions":[{"id":"https://openalex.org/I145872427","display_name":"Radboud University Nijmegen","ror":"https://ror.org/016xsfp80","country_code":"NL","type":"education","lineage":["https://openalex.org/I145872427"]}],"countries":["NL"],"is_corresponding":true,"raw_author_name":"Kristian Gonzalez Barman","raw_affiliation_strings":["Institute for Science in Society, Faculty of Science, Radboud University the Netherlands, Nijmegen, the Netherlands"],"affiliations":[{"raw_affiliation_string":"Institute for Science in Society, Faculty of Science, Radboud University the Netherlands, Nijmegen, the Netherlands","institution_ids":["https://openalex.org/I145872427"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100600328","display_name":"S. Caron","orcid":"https://orcid.org/0000-0003-2941-2829"},"institutions":[{"id":"https://openalex.org/I4210093566","display_name":"National Institute for Subatomic Physics","ror":"https://ror.org/00f9tz983","country_code":"NL","type":"facility","lineage":["https://openalex.org/I2800991832","https://openalex.org/I4210093566"]},{"id":"https://openalex.org/I145872427","display_name":"Radboud University Nijmegen","ror":"https://ror.org/016xsfp80","country_code":"NL","type":"education","lineage":["https://openalex.org/I145872427"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Sascha Caron","raw_affiliation_strings":["High Energy Physics, Faculty of Science, Radboud University the Netherlands, Nijmegen, the Netherlands","Nikhef, Science Park 105, Amsterdam, 1098 XG, the Netherlands"],"affiliations":[{"raw_affiliation_string":"High Energy Physics, Faculty of Science, Radboud University the Netherlands, Nijmegen, the Netherlands","institution_ids":["https://openalex.org/I145872427"]},{"raw_affiliation_string":"Nikhef, Science Park 105, Amsterdam, 1098 XG, the Netherlands","institution_ids":["https://openalex.org/I4210093566"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005492854","display_name":"Tom Claassen","orcid":null},"institutions":[{"id":"https://openalex.org/I145872427","display_name":"Radboud University Nijmegen","ror":"https://ror.org/016xsfp80","country_code":"NL","type":"education","lineage":["https://openalex.org/I145872427"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Tom Claassen","raw_affiliation_strings":["Institute for Computing and Information Sciences, Faculty of Science, Radboud University, Nijmegen, the Netherlands"],"affiliations":[{"raw_affiliation_string":"Institute for Computing and Information Sciences, Faculty of Science, Radboud University, Nijmegen, the Netherlands","institution_ids":["https://openalex.org/I145872427"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5078540559","display_name":"Henk de Regt","orcid":null},"institutions":[{"id":"https://openalex.org/I145872427","display_name":"Radboud University Nijmegen","ror":"https://ror.org/016xsfp80","country_code":"NL","type":"education","lineage":["https://openalex.org/I145872427"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Henk de Regt","raw_affiliation_strings":["Institute for Science in Society, Faculty of Science, Radboud University the Netherlands, Nijmegen, the Netherlands"],"affiliations":[{"raw_affiliation_string":"Institute for Science in Society, Faculty of Science, Radboud University the Netherlands, Nijmegen, the Netherlands","institution_ids":["https://openalex.org/I145872427"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5072006436"],"corresponding_institution_ids":["https://openalex.org/I145872427"],"apc_list":{"value":2390,"currency":"EUR","value_usd":2990},"apc_paid":{"value":2390,"currency":"EUR","value_usd":2990},"fwci":92.6018,"has_fulltext":true,"cited_by_count":15,"citation_normalized_percentile":{"value":0.99906522,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":"34","issue":"1","first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10778","display_name":"Philosophy and History of Science","score":0.9940000176429749,"subfield":{"id":"https://openalex.org/subfields/1207","display_name":"History and Philosophy of Science"},"field":{"id":"https://openalex.org/fields/12","display_name":"Arts and Humanities"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10778","display_name":"Philosophy and History of Science","score":0.9940000176429749,"subfield":{"id":"https://openalex.org/subfields/1207","display_name":"History and Philosophy of Science"},"field":{"id":"https://openalex.org/fields/12","display_name":"Arts and Humanities"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11883","display_name":"Embodied and Extended Cognition","score":0.9824000000953674,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T12002","display_name":"Computability, Logic, AI Algorithms","score":0.9621999859809875,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/philosophy-of-science","display_name":"Philosophy of science","score":0.7805852293968201},{"id":"https://openalex.org/keywords/philosophy-of-mind","display_name":"Philosophy of mind","score":0.6751915812492371},{"id":"https://openalex.org/keywords/theory-of-computation","display_name":"Theory of computation","score":0.6649566888809204},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.6174004673957825},{"id":"https://openalex.org/keywords/cognitive-science","display_name":"Cognitive science","score":0.4875630736351013},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.4499658942222595},{"id":"https://openalex.org/keywords/epistemology","display_name":"Epistemology","score":0.38654565811157227},{"id":"https://openalex.org/keywords/management-science","display_name":"Management science","score":0.3569999039173126},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.27903276681900024},{"id":"https://openalex.org/keywords/philosophy","display_name":"Philosophy","score":0.2547236382961273},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.16197237372398376},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.11197087168693542},{"id":"https://openalex.org/keywords/metaphysics","display_name":"Metaphysics","score":0.10676345229148865},{"id":"https://openalex.org/keywords/geography","display_name":"Geography","score":0.07511252164840698}],"concepts":[{"id":"https://openalex.org/C110099512","wikidata":"https://www.wikidata.org/wiki/Q59115","display_name":"Philosophy of science","level":2,"score":0.7805852293968201},{"id":"https://openalex.org/C24726157","wikidata":"https://www.wikidata.org/wiki/Q23407","display_name":"Philosophy of mind","level":3,"score":0.6751915812492371},{"id":"https://openalex.org/C24858836","wikidata":"https://www.wikidata.org/wiki/Q844718","display_name":"Theory of computation","level":2,"score":0.6649566888809204},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6174004673957825},{"id":"https://openalex.org/C188147891","wikidata":"https://www.wikidata.org/wiki/Q147638","display_name":"Cognitive science","level":1,"score":0.4875630736351013},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4499658942222595},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.38654565811157227},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.3569999039173126},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.27903276681900024},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.2547236382961273},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.16197237372398376},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.11197087168693542},{"id":"https://openalex.org/C182744844","wikidata":"https://www.wikidata.org/wiki/Q35277","display_name":"Metaphysics","level":2,"score":0.10676345229148865},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.07511252164840698},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1007/s11023-024-09657-1","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s11023-024-09657-1","pdf_url":"https://link.springer.com/content/pdf/10.1007/s11023-024-09657-1.pdf","source":{"id":"https://openalex.org/S160466889","display_name":"Minds and Machines","issn_l":"0924-6495","issn":["0924-6495","1572-8641"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Minds and Machines","raw_type":"journal-article"},{"id":"pmh:oai:archive.ugent.be:01JJ1QZQTQFM2AN9N36DT5FD3E","is_oa":true,"landing_page_url":"http://hdl.handle.net/1854/LU-01JJ1QZQTQFM2AN9N36DT5FD3E","pdf_url":"https://biblio.ugent.be/publication/01JJ1QZQTQFM2AN9N36DT5FD3E/file/01JJ1R2Y2HZYN8435MV9SEZ6MF.pdf","source":{"id":"https://openalex.org/S4306400478","display_name":"Ghent University Academic Bibliography (Ghent University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I32597200","host_organization_name":"Ghent University","host_organization_lineage":["https://openalex.org/I32597200"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"ISSN: 1572-8641","raw_type":"info:eu-repo/semantics/publishedVersion"},{"id":"pmh:oai:repository.ubn.ru.nl:2066/306210","is_oa":true,"landing_page_url":"https://repository.ubn.ru.nl/handle/2066/306210","pdf_url":"https://repository.ubn.ru.nl/bitstream/handle/2066/306210/1/306210.pdf","source":{"id":"https://openalex.org/S4306401067","display_name":"Radboud Repository (Radboud University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I145872427","host_organization_name":"Radboud University Nijmegen","host_organization_lineage":["https://openalex.org/I145872427"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Minds and Machines, 34, 1, pp. 1-16","raw_type":"Article / Letter to editor"}],"best_oa_location":{"id":"doi:10.1007/s11023-024-09657-1","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s11023-024-09657-1","pdf_url":"https://link.springer.com/content/pdf/10.1007/s11023-024-09657-1.pdf","source":{"id":"https://openalex.org/S160466889","display_name":"Minds and Machines","issn_l":"0924-6495","issn":["0924-6495","1572-8641"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Minds and Machines","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4395467630.pdf"},"referenced_works_count":52,"referenced_works":["https://openalex.org/W174297834","https://openalex.org/W393180982","https://openalex.org/W1916021481","https://openalex.org/W1983870751","https://openalex.org/W1998413842","https://openalex.org/W2001771035","https://openalex.org/W2011122440","https://openalex.org/W2038906023","https://openalex.org/W2068617190","https://openalex.org/W2076300962","https://openalex.org/W2084092936","https://openalex.org/W2106852830","https://openalex.org/W2140107468","https://openalex.org/W2158831831","https://openalex.org/W2176526532","https://openalex.org/W2251410821","https://openalex.org/W2413604966","https://openalex.org/W2491030098","https://openalex.org/W2577494375","https://openalex.org/W2754527400","https://openalex.org/W2883158190","https://openalex.org/W2886080749","https://openalex.org/W2887812497","https://openalex.org/W2916886569","https://openalex.org/W2923014074","https://openalex.org/W2962717047","https://openalex.org/W2963033005","https://openalex.org/W2999905431","https://openalex.org/W3084902313","https://openalex.org/W3096718571","https://openalex.org/W3133702157","https://openalex.org/W3160638507","https://openalex.org/W3192253141","https://openalex.org/W3192862352","https://openalex.org/W3209295695","https://openalex.org/W4210828651","https://openalex.org/W4233343253","https://openalex.org/W4239937942","https://openalex.org/W4245724339","https://openalex.org/W4246671941","https://openalex.org/W4254080801","https://openalex.org/W4281486074","https://openalex.org/W4290634670","https://openalex.org/W4304185479","https://openalex.org/W4307833521","https://openalex.org/W4318479436","https://openalex.org/W4360957277","https://openalex.org/W6635469476","https://openalex.org/W6677916085","https://openalex.org/W6680889820","https://openalex.org/W6739901393","https://openalex.org/W6778883912"],"related_works":["https://openalex.org/W2531287006","https://openalex.org/W4220673763","https://openalex.org/W2037107578","https://openalex.org/W2078814290","https://openalex.org/W1965227077","https://openalex.org/W2052861746","https://openalex.org/W2043402388","https://openalex.org/W2100605390","https://openalex.org/W2953698770","https://openalex.org/W109703496"],"abstract_inverted_index":{"Abstract":[0],"Scientific":[1,138],"understanding":[2,20,78,95,182,191],"is":[3,11,36],"a":[4,32,52,56,69,98,137,144,160,171],"fundamental":[5],"goal":[6],"of":[7,21,44,65,72,93,100,106,146,155],"science.":[8,66],"However,":[9],"there":[10],"currently":[12],"no":[13],"good":[14],"way":[15],"to":[16,38,54,75,85,114,117,124,194],"measure":[17],"the":[18,112,122,151],"scientific":[19,45,59,94,107,156,181,190],"agents,":[22],"whether":[23],"these":[24,147],"be":[25,80,129],"humans":[26],"or":[27],"Artificial":[28],"Intelligence":[29],"systems.":[30],"Without":[31],"clear":[33],"benchmark,":[34],"it":[35],"challenging":[37],"evaluate":[39],"and":[40,121,153,169,179,192],"compare":[41],"different":[42,104,130,132],"levels":[43,105],"understanding.":[46,157],"In":[47],"this":[48,91],"paper,":[49],"we":[50,183],"propose":[51],"framework":[53],"create":[55],"benchmark":[57],"for":[58,150,173],"understanding,":[60,73,108],"utilizing":[61],"tools":[62],"from":[63],"philosophy":[64],"We":[67,89,134],"adopt":[68],"behavioral":[70],"conception":[71],"according":[74],"which":[76],"genuine":[77],"should":[79],"recognized":[81],"as":[82],"an":[83,119],"ability":[84,123],"perform":[86],"certain":[87],"tasks.":[88],"extend":[90],"notion":[92],"by":[96,143],"considering":[97],"set":[99,145],"questions":[101],"that":[102],"gauge":[103],"covering":[109],"information":[110,116],"retrieval,":[111],"capability":[113],"arrange":[115],"produce":[118],"explanation,":[120],"infer":[125],"how":[126],"things":[127],"would":[128],"under":[131],"circumstances.":[133],"suggest":[135],"building":[136],"Understanding":[139],"Benchmark":[140],"(SUB),":[141],"formed":[142],"tests,":[148],"allowing":[149],"evaluation":[152],"comparison":[154],"Benchmarking":[158],"plays":[159],"crucial":[161],"role":[162],"in":[163],"establishing":[164],"trust,":[165],"ensuring":[166],"quality":[167],"control,":[168],"providing":[170],"basis":[172],"performance":[174],"evaluation.":[175],"By":[176],"aligning":[177],"machine":[178],"human":[180],"can":[184],"improve":[185],"their":[186],"utility,":[187],"ultimately":[188],"advancing":[189],"helping":[193],"discover":[195],"new":[196],"insights":[197],"within":[198],"machines.":[199]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":11},{"year":2024,"cited_by_count":1}],"updated_date":"2026-03-16T09:10:04.655348","created_date":"2025-10-10T00:00:00"}