{"id":"https://openalex.org/W7202391494","doi":"https://doi.org/10.5281/zenodo.21907639","title":"Auditing the evaluation pipeline of a machine learning benchmark: data, code and results for a case study on the Vesuvius Challenge","display_name":"Auditing the evaluation pipeline of a machine learning benchmark: data, code and results for a case study on the Vesuvius Challenge","publication_year":2026,"publication_date":"2026-08-12","ids":{"openalex":"https://openalex.org/W7202391494","doi":"https://doi.org/10.5281/zenodo.21907639"},"language":"en","primary_location":{"id":"doi:10.5281/zenodo.21907639","doi":"https://doi.org/10.5281/zenodo.21907639","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.21907639","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Software"},"volume":null,"type":"software","is_traditional":false,"indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.5281/zenodo.21907639","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5064019700","display_name":"Sedat Golgiyaz","orcid":"https://orcid.org/0000-0003-0305-9713","observed_orcids":["https://orcid.org/0000-0003-0305-9713"]},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"GOLGIYAZ, SEDAT","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0003-0305-9713","affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":["https://openalex.org/A5064019700"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.263700008392334,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.263700008392334,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.19900000095367432,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.026399999856948853,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/nnu-net","display_name":"nnU-Net","score":0.7129999995231628},{"id":"https://openalex.org/keywords/kaggle-competition","display_name":"Kaggle competition","score":0.5730000138282776}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6294000148773193},{"id":"https://openalex.org/C154504017","wikidata":"https://www.wikidata.org/wiki/Q853614","display_name":"Identifier","level":2,"score":0.5464000105857849},{"id":"https://openalex.org/C519991488","wikidata":"https://www.wikidata.org/wiki/Q28865","display_name":"Python (programming language)","level":2,"score":0.5361999869346619},{"id":"https://openalex.org/C61423126","wikidata":"https://www.wikidata.org/wiki/Q187432","display_name":"Scripting language","level":2,"score":0.5329999923706055},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4869999885559082},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.4587000012397766},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.4494999945163727},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4453999996185303},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.4431000053882599},{"id":"https://openalex.org/C199521495","wikidata":"https://www.wikidata.org/wiki/Q181487","display_name":"Audit","level":2,"score":0.43459999561309814},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.42289999127388},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3840999901294708},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.36419999599456787},{"id":"https://openalex.org/C20556612","wikidata":"https://www.wikidata.org/wiki/Q4469374","display_name":"Volume (thermodynamics)","level":2,"score":0.33719998598098755},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.3197999894618988},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.31279999017715454},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.3050000071525574},{"id":"https://openalex.org/C115901376","wikidata":"https://www.wikidata.org/wiki/Q184199","display_name":"Automation","level":2,"score":0.3041999936103821},{"id":"https://openalex.org/C113954288","wikidata":"https://www.wikidata.org/wiki/Q186885","display_name":"Timestamp","level":2,"score":0.2962999939918518},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.2874000072479248},{"id":"https://openalex.org/C175309249","wikidata":"https://www.wikidata.org/wiki/Q725864","display_name":"Pipeline transport","level":2,"score":0.287200003862381},{"id":"https://openalex.org/C8038995","wikidata":"https://www.wikidata.org/wiki/Q1152135","display_name":"Unsupervised learning","level":2,"score":0.2766999900341034},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.2720000147819519},{"id":"https://openalex.org/C168820333","wikidata":"https://www.wikidata.org/wiki/Q448889","display_name":"Visual inspection","level":2,"score":0.2703999876976013},{"id":"https://openalex.org/C53893814","wikidata":"https://www.wikidata.org/wiki/Q7378909","display_name":"Rule-based machine translation","level":2,"score":0.26759999990463257},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.2621000111103058},{"id":"https://openalex.org/C2776809875","wikidata":"https://www.wikidata.org/wiki/Q1375963","display_name":"Converse","level":2,"score":0.25459998846054077},{"id":"https://openalex.org/C151233233","wikidata":"https://www.wikidata.org/wiki/Q47574","display_name":"Units of measurement","level":2,"score":0.2517000138759613}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.5281/zenodo.21907639","doi":"https://doi.org/10.5281/zenodo.21907639","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.21907639","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Software"},{"id":"doi:10.5281/zenodo.21907640","doi":"https://doi.org/10.5281/zenodo.21907640","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.21907640","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Software"}],"best_oa_location":{"id":"doi:10.5281/zenodo.21907639","doi":"https://doi.org/10.5281/zenodo.21907639","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.21907639","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Software"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7160659278","https://openalex.org/W7221519538","https://openalex.org/W6949381743","https://openalex.org/W7213260235","https://openalex.org/W7204890779","https://openalex.org/W7155622900","https://openalex.org/W7171807093","https://openalex.org/W7128771738","https://openalex.org/W3034558499","https://openalex.org/W6928977325"],"abstract_inverted_index":{"Data,":[0],"code":[1],"and":[2,51,62,73,83,114,145,165,178,189,229],"results":[3,87],"accompanying":[4],"the":[5,8,20,26,30,41,57,79,85,91,94,99,111,125,142,146,150,157,161,168,171,198,202,210,222,240],"manuscript":[6],"\"Auditing":[7],"evaluation":[9],"pipeline":[10,44],"of":[11,29,56,124,201],"a":[12,16,121,128,193],"machine":[13],"learning":[14],"benchmark:":[15],"case":[17],"study":[18],"on":[19,90,174],"Vesuvius":[21],"Challenge\".":[22],"The":[23,107,183,205],"deposit":[24],"contains":[25],"final":[27],"leaderboard":[28,112,139],"competition":[31,151],"for":[32,66,135,149],"1,391":[33],"teams":[34],"with":[35,43,60,163],"both":[36],"scores;":[37],"47":[38],"submissions":[39],"by":[40],"author":[42],"provenance;":[45],"scroll-disjoint":[46],"fold":[47,184],"definitions":[48],"at":[49,221,239],"scroll":[50,187],"volume":[52,190],"level;":[53,242],"an":[54],"inventory":[55],"audited":[58],"notebooks":[59,155],"decision":[61],"rationale;":[63],"official-metric":[64,172],"scores":[65],"thirty":[67],"volumes":[68],"under":[69,197],"three":[70],"post-processing":[71],"reductions;":[72],"18":[74],"Python":[75],"scripts.":[76],"run_all.py":[77],"is":[78],"single":[80],"entry":[81],"point":[82],"reproduces":[84,160],"quantitative":[86],"that":[88],"rest":[89],"redistributed":[92],"data;":[93],"code-state":[95,129],"census":[96],"additionally":[97],"requires":[98],"30":[100],"inference":[101],"notebooks,":[102],"which":[103],"are":[104,180,238,244],"not":[105,181],"redistributed.":[106,182],"remaining":[108],"scripts":[109],"cover":[110],"analysis":[113,162],"its":[115],"coupling":[116],"diagnostics,":[117],"exact":[118],"group-coverage":[119],"probabilities,":[120],"numerical":[122],"check":[123],"threshold-invariance":[126],"result,":[127],"census,":[130],"figure":[131],"generation,":[132],"split-regime":[133],"generators":[134],"nnU-Net,":[136],"index":[137],"reconciliation,":[138],"reconstruction":[140],"from":[141,209],"platform":[143],"exports,":[144],"installation":[147],"sequence":[148],"scoring":[152],"package.":[153],"Two":[154],"accompany":[156],"scripts:":[158],"one":[159],"tables":[164],"figures":[166],"inline,":[167],"other":[169],"performs":[170],"measurement":[173],"Kaggle.":[175],"Competition":[176],"imagery":[177],"annotations":[179],"files":[185,237],"contain":[186],"membership":[188],"identifiers":[191],"only,":[192],"derived":[194],"tabulation":[195],"shared":[196],"non-commercial":[199],"terms":[200],"source":[203],"data.":[204],"imaging":[206],"data":[207],"derive":[208],"EduceLab-Scrolls":[211],"dataset":[212],"(Parsons":[213],"et":[214],"al.":[215],"2023,":[216],"10.48550/arXiv.2304.02084);":[217],"acquisitions":[218],"were":[219],"made":[220],"European":[223],"Synchrotron":[224],"Radiation":[225],"Facility":[226],"(beamline":[227,233],"BM18)":[228],"Diamond":[230],"Light":[231],"Source":[232],"I12).":[234],"All":[235],"35":[236],"top":[241],"there":[243],"no":[245],"subdirectories.":[246]},"counts_by_year":[],"updated_date":"2026-10-09T09:14:12.587421","created_date":"2026-08-14T00:00:00"}